Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1052C200B49 for ; Wed, 20 Jul 2016 00:01:51 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 0F74C160A5C; Tue, 19 Jul 2016 22:01:51 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 90447160A8C for ; Wed, 20 Jul 2016 00:01:49 +0200 (CEST) Received: (qmail 1681 invoked by uid 500); 19 Jul 2016 22:01:42 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 446 invoked by uid 99); 19 Jul 2016 22:01:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2016 22:01:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 67E6DE0B40; Tue, 19 Jul 2016 22:01:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: subru@apache.org To: common-commits@hadoop.apache.org Date: Tue, 19 Jul 2016 22:02:00 -0000 Message-Id: <1682c38efdfd48ee9415eba64029c3db@git.apache.org> In-Reply-To: <34f611f4ac964d3fb5185d84fa54c7b3@git.apache.org> References: <34f611f4ac964d3fb5185d84fa54c7b3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/50] hadoop git commit: YARN-5298. Mount usercache and NM filecache directories into Docker container. Contributed by Sidharta Seethana. archived-at: Tue, 19 Jul 2016 22:01:51 -0000 YARN-5298. Mount usercache and NM filecache directories into Docker container. Contributed by Sidharta Seethana. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58e18508 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58e18508 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58e18508 Branch: refs/heads/YARN-2915 Commit: 58e18508018081b5b5aa7c12cc5af386146cd26b Parents: be26c1b Author: Varun Vasudev Authored: Thu Jul 14 16:11:07 2016 +0530 Committer: Varun Vasudev Committed: Thu Jul 14 16:11:07 2016 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/api/ApplicationConstants.java | 6 +++ .../nodemanager/LinuxContainerExecutor.java | 5 +- .../launcher/ContainerLaunch.java | 43 +++++++++++++++- .../runtime/DockerLinuxContainerRuntime.java | 7 +++ .../runtime/LinuxContainerRuntimeConstants.java | 4 ++ .../executor/ContainerStartContext.java | 29 ++++++++++- .../TestLinuxContainerExecutorWithMocks.java | 8 +++ .../launcher/TestContainerLaunch.java | 9 ++-- .../runtime/TestDockerContainerRuntime.java | 52 +++++++++++++------- 9 files changed, 136 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index fa475e7..9442c53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -230,6 +230,12 @@ public interface ApplicationConstants { LOCAL_DIRS("LOCAL_DIRS"), /** + * $LOCAL_USER_DIRS + * Final, exported by NodeManager and non-modifiable by users. + */ + LOCAL_USER_DIRS("LOCAL_USER_DIRS"), + + /** * $LOG_DIRS * Final, exported by NodeManager and non-modifiable by users. * Comma separate list of directories that the container should use for http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 1072b5a..be5d078 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler; import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler; import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler; -import org.apache.hadoop.yarn.util.ConverterUtils; import java.io.File; import java.io.IOException; @@ -311,6 +310,8 @@ public class LinuxContainerExecutor extends ContainerExecutor { Path containerWorkDir = ctx.getContainerWorkDir(); List localDirs = ctx.getLocalDirs(); List logDirs = ctx.getLogDirs(); + List filecacheDirs = ctx.getFilecacheDirs(); + List userLocalDirs = ctx.getUserLocalDirs(); List containerLocalDirs = ctx.getContainerLocalDirs(); List containerLogDirs = ctx.getContainerLogDirs(); Map> localizedResources = ctx.getLocalizedResources(); @@ -397,6 +398,8 @@ public class LinuxContainerExecutor extends ContainerExecutor { .setExecutionAttribute(PID_FILE_PATH, pidFilePath) .setExecutionAttribute(LOCAL_DIRS, localDirs) .setExecutionAttribute(LOG_DIRS, logDirs) + .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs) + .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs) .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs) .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs) .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions); http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 7e9030c..dffe120 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -230,6 +230,8 @@ public class ContainerLaunch implements Callable { pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath); List localDirs = dirsHandler.getLocalDirs(); List logDirs = dirsHandler.getLogDirs(); + List filecacheDirs = getNMFilecacheDirs(localDirs); + List userLocalDirs = getUserLocalDirs(localDirs); List containerLocalDirs = getContainerLocalDirs(localDirs); List containerLogDirs = getContainerLogDirs(logDirs); @@ -242,6 +244,7 @@ public class ContainerLaunch implements Callable { try { // /////////// Write out the container-script in the nmPrivate space. List appDirs = new ArrayList(localDirs.size()); + for (String localDir : localDirs) { Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE); Path userdir = new Path(usersdir, user); @@ -258,7 +261,8 @@ public class ContainerLaunch implements Callable { new Path(containerWorkDir, FINAL_CONTAINER_TOKENS_FILE).toUri().getPath()); // Sanitize the container's environment - sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs, + sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs, + containerLogDirs, localResources, nmPrivateClasspathJarDir); // Write out the environment @@ -288,6 +292,8 @@ public class ContainerLaunch implements Callable { .setContainerWorkDir(containerWorkDir) .setLocalDirs(localDirs) .setLogDirs(logDirs) + .setFilecacheDirs(filecacheDirs) + .setUserLocalDirs(userLocalDirs) .setContainerLocalDirs(containerLocalDirs) .setContainerLogDirs(containerLogDirs) .build()); @@ -353,6 +359,35 @@ public class ContainerLaunch implements Callable { return containerLocalDirs; } + protected List getUserLocalDirs(List localDirs) { + List userLocalDirs = new ArrayList<>(localDirs.size()); + String user = container.getUser(); + + for (String localDir : localDirs) { + String userLocalDir = localDir + Path.SEPARATOR + + ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + + Path.SEPARATOR; + + userLocalDirs.add(userLocalDir); + } + + return userLocalDirs; + } + + protected List getNMFilecacheDirs(List localDirs) { + List filecacheDirs = new ArrayList<>(localDirs.size()); + + for (String localDir : localDirs) { + String filecacheDir = localDir + Path.SEPARATOR + + ContainerLocalizer.FILECACHE; + + filecacheDirs.add(filecacheDir); + } + + return filecacheDirs; + } + + protected Map> getLocalizedResources() throws YarnException { Map> localResources = container.getLocalizedResources(); @@ -990,7 +1025,8 @@ public class ContainerLaunch implements Callable { } public void sanitizeEnv(Map environment, Path pwd, - List appDirs, List containerLogDirs, + List appDirs, List userLocalDirs, List + containerLogDirs, Map> resources, Path nmPrivateClasspathJarDir) throws IOException { /** @@ -1012,6 +1048,9 @@ public class ContainerLaunch implements Callable { environment.put(Environment.LOCAL_DIRS.name(), StringUtils.join(",", appDirs)); + environment.put(Environment.LOCAL_USER_DIRS.name(), StringUtils.join(",", + userLocalDirs)); + environment.put(Environment.LOG_DIRS.name(), StringUtils.join(",", containerLogDirs)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index e1a162e..a1606f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -318,6 +318,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @SuppressWarnings("unchecked") List logDirs = ctx.getExecutionAttribute(LOG_DIRS); @SuppressWarnings("unchecked") + List filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS); + @SuppressWarnings("unchecked") List containerLocalDirs = ctx.getExecutionAttribute( CONTAINER_LOCAL_DIRS); @SuppressWarnings("unchecked") @@ -326,6 +328,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @SuppressWarnings("unchecked") Map> localizedResources = ctx.getExecutionAttribute( LOCALIZED_RESOURCES); + @SuppressWarnings("unchecked") + List userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS); + Set capabilities = new HashSet<>(Arrays.asList(conf.getStrings( YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES))); @@ -340,8 +345,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { .addMountLocation("/etc/passwd", "/etc/password:ro"); List allDirs = new ArrayList<>(containerLocalDirs); + allDirs.addAll(filecacheDirs); allDirs.add(containerWorkDir.toString()); allDirs.addAll(containerLogDirs); + allDirs.addAll(userLocalDirs); for (String dir: allDirs) { runCommand.addMountLocation(dir, dir); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java index 96ff40d..0c1ec3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java @@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants { List.class, "local_dirs"); public static final Attribute LOG_DIRS = Attribute.attribute( List.class, "log_dirs"); + public static final Attribute FILECACHE_DIRS = Attribute.attribute( + List.class, "filecache_dirs"); + public static final Attribute USER_LOCAL_DIRS = Attribute.attribute( + List.class, "user_local_dirs"); public static final Attribute CONTAINER_LOCAL_DIRS = Attribute .attribute(List.class, "container_local_dirs"); public static final Attribute CONTAINER_LOG_DIRS = Attribute.attribute( http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java index b14a520..9c5f3a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java @@ -45,6 +45,8 @@ public final class ContainerStartContext { private final Path containerWorkDir; private final List localDirs; private final List logDirs; + private final List filecacheDirs; + private final List userLocalDirs; private final List containerLocalDirs; private final List containerLogDirs; @@ -58,6 +60,8 @@ public final class ContainerStartContext { private Path containerWorkDir; private List localDirs; private List logDirs; + private List filecacheDirs; + private List userLocalDirs; private List containerLocalDirs; private List containerLogDirs; @@ -111,6 +115,16 @@ public final class ContainerStartContext { return this; } + public Builder setFilecacheDirs(List filecacheDirs) { + this.filecacheDirs = filecacheDirs; + return this; + } + + public Builder setUserLocalDirs(List userLocalDirs) { + this.userLocalDirs = userLocalDirs; + return this; + } + public Builder setContainerLocalDirs(List containerLocalDirs) { this.containerLocalDirs = containerLocalDirs; return this; @@ -136,6 +150,8 @@ public final class ContainerStartContext { this.containerWorkDir = builder.containerWorkDir; this.localDirs = builder.localDirs; this.logDirs = builder.logDirs; + this.filecacheDirs = builder.filecacheDirs; + this.userLocalDirs = builder.userLocalDirs; this.containerLocalDirs = builder.containerLocalDirs; this.containerLogDirs = builder.containerLogDirs; } @@ -180,11 +196,20 @@ public final class ContainerStartContext { return Collections.unmodifiableList(this.logDirs); } + public List getFilecacheDirs() { + return Collections.unmodifiableList(this.filecacheDirs); + } + + public List getUserLocalDirs() { + return Collections.unmodifiableList(this.userLocalDirs); + } + public List getContainerLocalDirs() { - return this.containerLocalDirs; + return Collections.unmodifiableList(this.containerLocalDirs); } public List getContainerLogDirs() { - return this.containerLogDirs; + return Collections.unmodifiableList(this + .containerLogDirs); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 0ef788b..aae0037 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -180,6 +180,10 @@ public class TestLinuxContainerExecutorWithMocks { .setContainerWorkDir(workDir) .setLocalDirs(dirsHandler.getLocalDirs()) .setLogDirs(dirsHandler.getLogDirs()) + .setFilecacheDirs(new ArrayList<>()) + .setUserLocalDirs(new ArrayList<>()) + .setContainerLocalDirs(new ArrayList<>()) + .setContainerLogDirs(new ArrayList<>()) .build()); assertEquals(0, ret); assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, @@ -345,6 +349,10 @@ public class TestLinuxContainerExecutorWithMocks { .setContainerWorkDir(workDir) .setLocalDirs(dirsHandler.getLocalDirs()) .setLogDirs(dirsHandler.getLogDirs()) + .setFilecacheDirs(new ArrayList<>()) + .setUserLocalDirs(new ArrayList<>()) + .setContainerLocalDirs(new ArrayList<>()) + .setContainerLogDirs(new ArrayList<>()) .build()); Assert.assertNotSame(0, ret); http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index cc9e662..6b32dd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -455,6 +455,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { "target/test-dir"); Path pwd = new Path(testDir); List appDirs = new ArrayList(); + List userLocalDirs = new ArrayList<>(); List containerLogs = new ArrayList(); Map> resources = new HashMap>(); @@ -465,8 +466,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { Path nmp = new Path(testDir); - launch.sanitizeEnv( - userSetEnv, pwd, appDirs, containerLogs, resources, nmp); + launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs, + resources, nmp); List result = getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name())); @@ -484,8 +485,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager); - launch.sanitizeEnv( - userSetEnv, pwd, appDirs, containerLogs, resources, nmp); + launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs, + resources, nmp); result = getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/58e18508/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 3c0b67d..bb3f0f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -82,6 +82,8 @@ public class TestDockerContainerRuntime { private Path pidFilePath; private List localDirs; private List logDirs; + private List filecacheDirs; + private List userLocalDirs; private List containerLocalDirs; private List containerLogDirs; private Map> localizedResources; @@ -126,13 +128,17 @@ public class TestDockerContainerRuntime { pidFilePath = new Path("/test_pid_file_path"); localDirs = new ArrayList<>(); logDirs = new ArrayList<>(); + filecacheDirs = new ArrayList<>(); resourcesOptions = "cgroups=none"; + userLocalDirs = new ArrayList<>(); containerLocalDirs = new ArrayList<>(); containerLogDirs = new ArrayList<>(); localizedResources = new HashMap<>(); localDirs.add("/test_local_dir"); logDirs.add("/test_log_dir"); + filecacheDirs.add("/test_filecache_dir"); + userLocalDirs.add("/test_user_local_dir"); containerLocalDirs.add("/test_container_local_dir"); containerLogDirs.add("/test_container_log_dir"); localizedResources.put(new Path("/test_local_dir/test_resource_file"), @@ -156,6 +162,8 @@ public class TestDockerContainerRuntime { .setExecutionAttribute(PID_FILE_PATH, pidFilePath) .setExecutionAttribute(LOCAL_DIRS, localDirs) .setExecutionAttribute(LOG_DIRS, logDirs) + .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs) + .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs) .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs) .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs) .setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources) @@ -262,12 +270,15 @@ public class TestDockerContainerRuntime { .append("-v %4$s:%4$s ") .append("-v %5$s:%5$s ") .append("-v %6$s:%6$s ") - .append("%7$s ") - .append("bash %8$s/launch_container.sh"); + .append("-v %7$s:%7$s ") + .append("-v %8$s:%8$s ").append("%9$s ") + .append("bash %10$s/launch_container.sh"); - String expectedCommand = String.format(expectedCommandTemplate.toString(), - containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), - containerWorkDir, containerLogDirs.get(0), image, containerWorkDir); + String expectedCommand = String + .format(expectedCommandTemplate.toString(), containerId, runAsUser, + containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), + containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), + image, containerWorkDir); List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); @@ -363,13 +374,15 @@ public class TestDockerContainerRuntime { .append(getExpectedTestCapabilitiesArgumentString()) .append("-v /etc/passwd:/etc/password:ro ") .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") - .append("-v %6$s:%6$s ").append("%7$s ") - .append("bash %8$s/launch_container.sh"); + .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ") + .append("-v %8$s:%8$s ").append("%9$s ") + .append("bash %10$s/launch_container.sh"); String expectedCommand = String .format(expectedCommandTemplate.toString(), containerId, runAsUser, - containerWorkDir, containerLocalDirs.get(0), containerWorkDir, - containerLogDirs.get(0), image, containerWorkDir); + containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), + containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), + image, containerWorkDir); List dockerCommands = Files .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); @@ -416,13 +429,15 @@ public class TestDockerContainerRuntime { .append(getExpectedTestCapabilitiesArgumentString()) .append("-v /etc/passwd:/etc/password:ro ") .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") - .append("-v %6$s:%6$s ").append("%7$s ") - .append("bash %8$s/launch_container.sh"); + .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ") + .append("-v %8$s:%8$s ").append("%9$s ") + .append("bash %10$s/launch_container.sh"); String expectedCommand = String .format(expectedCommandTemplate.toString(), containerId, runAsUser, - containerWorkDir, containerLocalDirs.get(0), containerWorkDir, - containerLogDirs.get(0), image, containerWorkDir); + containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), + containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), + image, containerWorkDir); List dockerCommands = Files .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); @@ -451,14 +466,15 @@ public class TestDockerContainerRuntime { .append(getExpectedTestCapabilitiesArgumentString()) .append("-v /etc/passwd:/etc/password:ro ") .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") - .append("-v %6$s:%6$s ").append("%7$s ") - .append("bash %8$s/launch_container.sh"); + .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ") + .append("-v %8$s:%8$s ").append("%9$s ") + .append("bash %10$s/launch_container.sh"); expectedCommand = String .format(expectedCommandTemplate.toString(), containerId, runAsUser, - containerWorkDir, containerLocalDirs.get(0), containerWorkDir, - containerLogDirs.get(0), image, containerWorkDir); - + containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), + containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), + image, containerWorkDir); dockerCommands = Files .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org