From common-commits-return-93116-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Feb 12 21:19:07 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 0D914180718 for ; Tue, 12 Feb 2019 22:19:06 +0100 (CET) Received: (qmail 50160 invoked by uid 500); 12 Feb 2019 21:19:01 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 50144 invoked by uid 99); 12 Feb 2019 21:19:01 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Feb 2019 21:19:01 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 6095582A2A; Tue, 12 Feb 2019 21:19:00 +0000 (UTC) Date: Tue, 12 Feb 2019 21:19:00 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch trunk updated: YARN-9184. Add a system flag to allow update to latest docker images. Contributed by Zhaohui Xin MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155000634023.22372.970999999374817178@gitbox.apache.org> From: eyang@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/trunk X-Git-Reftype: branch X-Git-Oldrev: 7806403842ddd0f5b339e3dca42688b970cae267 X-Git-Newrev: 3dc252326693170ac1b31bf2914bae72ca73d31a X-Git-Rev: 3dc252326693170ac1b31bf2914bae72ca73d31a X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. eyang pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 3dc2523 YARN-9184. Add a system flag to allow update to latest docker images. Contributed by Zhaohui Xin 3dc2523 is described below commit 3dc252326693170ac1b31bf2914bae72ca73d31a Author: Eric Yang AuthorDate: Tue Feb 12 16:16:35 2019 -0500 YARN-9184. Add a system flag to allow update to latest docker images. Contributed by Zhaohui Xin --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 + .../src/main/resources/yarn-default.xml | 7 ++ .../linux/runtime/DockerLinuxContainerRuntime.java | 29 +++++- .../linux/runtime/TestDockerContainerRuntime.java | 115 +++++++++++++++++++++ .../src/site/markdown/DockerContainers.md | 9 ++ 5 files changed, 163 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 04d0fd1..4cd4cca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1936,6 +1936,10 @@ public class YarnConfiguration extends Configuration { public static final String NM_DOCKER_IMAGE_NAME = DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name"; + /** Default option to decide whether to pull the latest image or not. **/ + public static final String NM_DOCKER_IMAGE_UPDATE = + DOCKER_CONTAINER_RUNTIME_PREFIX + "image-update"; + /** Capabilities allowed (and added by default) for docker containers. **/ public static final String NM_DOCKER_CONTAINER_CAPABILITIES = DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index bfcbf4b..db29fb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1762,6 +1762,13 @@ + Default option to decide whether to pull the latest image + or not. + yarn.nodemanager.runtime.linux.docker.image-update + false + + + This configuration setting determines if privileged docker containers are allowed on this cluster. Privileged containers are granted the complete set of capabilities and are not subject to the limitations diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 7fc386d..f1da846 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime. import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerExecCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerPullCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStartCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand; @@ -272,6 +273,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private Map csiClients = new HashMap<>(); private PrivilegedOperationExecutor privilegedOperationExecutor; private String defaultImageName; + private Boolean defaultImageUpdate; private Set allowedNetworks = new HashSet<>(); private String defaultNetwork; private CGroupsHandler cGroupsHandler; @@ -352,6 +354,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { defaultTmpfsMounts.clear(); defaultImageName = conf.getTrimmed( YarnConfiguration.NM_DOCKER_IMAGE_NAME, ""); + defaultImageUpdate = conf.getBoolean( + YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -802,6 +806,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { throws ContainerExecutionException { Container container = ctx.getContainer(); ContainerId containerId = container.getContainerId(); + String containerIdStr = containerId.toString(); Map environment = container.getLaunchContext() .getEnvironment(); String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); @@ -822,7 +827,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { validateImageName(imageName); - String containerIdStr = containerId.toString(); + if (defaultImageUpdate) { + pullImageFromRemote(containerIdStr, imageName); + } + String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER); String dockerRunAsUser = runAsUser; Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR); @@ -1379,6 +1387,25 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } } + public void pullImageFromRemote(String containerIdStr, String imageName) + throws ContainerExecutionException { + long start = System.currentTimeMillis(); + DockerPullCommand dockerPullCommand = new DockerPullCommand(imageName); + LOG.debug("now pulling docker image." + " image name: " + imageName + "," + + " container: " + containerIdStr); + + DockerCommandExecutor.executeDockerCommand(dockerPullCommand, + containerIdStr, null, + privilegedOperationExecutor, false, nmContext); + + long end = System.currentTimeMillis(); + long pullImageTimeMs = end - start; + LOG.debug("pull docker image done with " + + String.valueOf(pullImageTimeMs) + "ms spent." + + " image name: " + imageName + "," + + " container: " + containerIdStr); + } + private void executeLivelinessCheck(ContainerRuntimeContext ctx) throws ContainerExecutionException { String procFs = ctx.getExecutionAttribute(PROCFS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ef0f2e0..6669cac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -534,6 +534,121 @@ public class TestDockerContainerRuntime { } @Test + public void testDockerContainerLaunchWithoutDefaultImageUpdate() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + conf.setBoolean(YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false); + + runtime.initialize(conf, nmContext); + runtime.launchContainer(builder.build()); + List dockerCommands = readDockerCommands(); + Assert.assertEquals(false, + conf.getBoolean(YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false)); + + int expected = 13; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert + .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" mounts=" + + "/test_container_log_dir:/test_container_log_dir:rw," + + "/test_application_local_dir:/test_application_local_dir:rw," + + "/test_filecache_dir:/test_filecache_dir:ro," + + "/test_user_filecache_dir:/test_user_filecache_dir:ro", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDockerContainerLaunchWithDefaultImageUpdate() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + conf.setBoolean(YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, true); + + runtime.initialize(conf, nmContext); + runtime.launchContainer(builder.build()); + + ArgumentCaptor opCaptor = ArgumentCaptor.forClass( + PrivilegedOperation.class); + + //Two invocations expected. + verify(mockExecutor, times(2)) + .executePrivilegedOperation(any(), opCaptor.capture(), any(), + any(), anyBoolean(), anyBoolean()); + + List allCaptures = opCaptor.getAllValues(); + + // pull image from remote hub firstly + PrivilegedOperation op = allCaptures.get(0); + Assert.assertEquals(PrivilegedOperation.OperationType + .RUN_DOCKER_CMD, op.getOperationType()); + + File commandFile = new File(StringUtils.join(",", op.getArguments())); + FileInputStream fileInputStream = new FileInputStream(commandFile); + String fileContent = new String(IOUtils.toByteArray(fileInputStream)); + Assert.assertEquals("[docker-command-execution]\n" + + " docker-command=pull\n" + + " image=busybox:latest\n", fileContent); + fileInputStream.close(); + + // launch docker container + List dockerCommands = readDockerCommands(2); + + int expected = 13; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert + .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" mounts=" + + "/test_container_log_dir:/test_container_log_dir:rw," + + "/test_application_local_dir:/test_application_local_dir:rw," + + "/test_filecache_dir:/test_filecache_dir:ro," + + "/test_user_filecache_dir:/test_user_filecache_dir:ro", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test public void testContainerLaunchWithUserRemapping() throws ContainerExecutionException, PrivilegedOperationException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 8797cb5..135a0fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -132,6 +132,15 @@ The following properties should be set in yarn-site.xml: + yarn.nodemanager.runtime.linux.docker.image-update + false + + Optional. Default option to decide whether to pull the latest image + or not. + + + + yarn.nodemanager.runtime.linux.docker.allowed-container-networks host,none,bridge --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org