hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [12/50] [abbrv] hadoop git commit: YARN-8342. Enable untrusted docker image to run with launch command. Contributed by Eric Yang
Date Tue, 05 Jun 2018 19:41:30 GMT
YARN-8342. Enable untrusted docker image to run with launch command. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31998643
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31998643
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31998643

Branch: refs/heads/YARN-1011
Commit: 31998643a51f1e08f723f18dc5476ac1512d5b81
Parents: 8261f9e
Author: Billie Rinaldi <billie@apache.org>
Authored: Sat Jun 2 14:46:32 2018 -0700
Committer: Billie Rinaldi <billie@apache.org>
Committed: Sat Jun 2 14:46:32 2018 -0700

----------------------------------------------------------------------
 .../hadoop-yarn/conf/yarn-env.sh                |  1 +
 .../provider/docker/DockerProviderService.java  | 24 ++++++++--
 .../runtime/DockerLinuxContainerRuntime.java    | 23 +++++++++-
 .../container-executor/impl/utils/docker-util.c |  7 +--
 .../test/utils/test_docker_util.cc              | 48 ++++++++++----------
 .../src/site/markdown/DockerContainers.md       | 33 ++++++++++++--
 .../src/site/markdown/yarn-service/Examples.md  | 37 ++++++++++++++-
 .../markdown/yarn-service/YarnServiceAPI.md     |  2 +-
 8 files changed, 135 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index d865023..76d1d6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -166,3 +166,4 @@
 ###
 # Directory containing service examples
 # export YARN_SERVICE_EXAMPLES_DIR = $HADOOP_YARN_HOME/share/hadoop/yarn/yarn-service-examples
+# export YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
index 821682d..071b30a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -50,6 +50,26 @@ public class DockerProviderService extends AbstractProviderService
         compInstance.getCompSpec().getRunPrivilegedContainer());
   }
 
+  /**
+   * Check if system is default to disable docker override or
+   * user requested a Docker container with ENTRY_POINT support.
+   *
+   * @param component - YARN Service component
+   * @return true if Docker launch command override is disabled
+   */
+  private boolean checkUseEntryPoint(Component component) {
+    boolean overrideDisable = false;
+    String overrideDisableKey = Environment.
+        YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.
+            name();
+    String overrideDisableValue = (component
+        .getConfiguration().getEnv(overrideDisableKey) != null) ?
+            component.getConfiguration().getEnv(overrideDisableKey) :
+                System.getenv(overrideDisableKey);
+    overrideDisable = Boolean.parseBoolean(overrideDisableValue);
+    return overrideDisable;
+  }
+
   @Override
   public void buildContainerLaunchCommand(AbstractLauncher launcher,
       Service service, ComponentInstance instance,
@@ -58,9 +78,7 @@ public class DockerProviderService extends AbstractProviderService
       Map<String, String> tokensForSubstitution)
           throws IOException, SliderException {
     Component component = instance.getComponent().getComponentSpec();
-    boolean useEntryPoint = Boolean.parseBoolean(component
-        .getConfiguration().getEnv(Environment
-          .YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.name()));
+    boolean useEntryPoint = checkUseEntryPoint(component);
     if (useEntryPoint) {
       String launchCommand = component.getLaunchCommand();
       if (!StringUtils.isEmpty(launchCommand)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index fc095d5..e19379f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommand;
@@ -724,6 +725,25 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime
{
     return id;
   }
 
+  /**
+   * Check if system is default to disable docker override or
+   * user requested a Docker container with ENTRY_POINT support.
+   *
+   * @param environment - Docker container environment variables
+   * @return true if Docker launch command override is disabled
+   */
+  private boolean checkUseEntryPoint(Map<String, String> environment) {
+    boolean overrideDisable = false;
+    String overrideDisableKey = Environment.
+        YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.
+            name();
+    String overrideDisableValue = (environment.get(overrideDisableKey) != null)
+        ? environment.get(overrideDisableKey) :
+            System.getenv(overrideDisableKey);
+    overrideDisable = Boolean.parseBoolean(overrideDisableValue);
+    return overrideDisable;
+  }
+
   @Override
   public void launchContainer(ContainerRuntimeContext ctx)
       throws ContainerExecutionException {
@@ -734,8 +754,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime
{
     String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
     String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
     String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
-    boolean useEntryPoint = Boolean.parseBoolean(environment
-              .get(ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE));
+    boolean useEntryPoint = checkUseEntryPoint(environment);
 
     if(network == null || network.isEmpty()) {
       network = defaultNetwork;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index d34a5b2..ffc349a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -114,7 +114,7 @@ int check_trusted_image(const struct configuration *command_config, const
struct
   int i = 0;
   int ret = 0;
   char *image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, command_config);
-  char **privileged_registry = get_configuration_values_delimiter("docker.privileged-containers.registries",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+  char **privileged_registry = get_configuration_values_delimiter("docker.trusted.registries",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
   char *registry_ptr = NULL;
   if (image_name == NULL) {
     ret = INVALID_DOCKER_IMAGE_NAME;
@@ -1097,7 +1097,6 @@ static int add_mounts(const struct configuration *command_config, const
struct c
   if (ro != 0) {
     ro_suffix = ":ro";
   }
-
   if (values != NULL) {
     // Disable mount volumes if image is not trusted.
     if (check_trusted_image(command_config, conf) != 0) {
@@ -1480,10 +1479,6 @@ int get_docker_run_command(const char *command_file, const struct configuration
 
   launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION,
&command_config,
                                                       ",");
-  if (check_trusted_image(&command_config, conf) != 0) {
-    launch_command = NULL;
-  }
-
   if (launch_command != NULL) {
     for (i = 0; launch_command[i] != NULL; ++i) {
       ret = add_to_args(args, launch_command[i]);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 613755c..cd671ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -639,9 +639,9 @@ namespace ContainerExecutor {
     struct configuration container_cfg, cmd_cfg;
     struct args buff = ARGS_INITIAL_VALUE;
     int ret = 0;
-    std::string container_executor_cfg_contents[] = {"[docker]\n  docker.privileged-containers.enabled=1\n
 docker.privileged-containers.registries=hadoop",
-                                                     "[docker]\n  docker.privileged-containers.enabled=true\n
 docker.privileged-containers.registries=hadoop",
-                                                     "[docker]\n  docker.privileged-containers.enabled=True\n
 docker.privileged-containers.registries=hadoop",
+    std::string container_executor_cfg_contents[] = {"[docker]\n  docker.privileged-containers.enabled=1\n
 docker.trusted.registries=hadoop",
+                                                     "[docker]\n  docker.privileged-containers.enabled=true\n
 docker.trusted.registries=hadoop",
+                                                     "[docker]\n  docker.privileged-containers.enabled=True\n
 docker.trusted.registries=hadoop",
                                                      "[docker]\n  docker.privileged-containers.enabled=0",
                                                      "[docker]\n  docker.privileged-containers.enabled=false",
                                                      "[docker]\n"};
@@ -727,7 +727,7 @@ namespace ContainerExecutor {
     int ret = 0;
     std::string container_executor_cfg_contents = "[docker]\n"
         "  docker.allowed.capabilities=CHROOT,MKNOD\n"
-        "  docker.privileged-containers.registries=hadoop\n";
+        "  docker.trusted.registries=hadoop\n";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n  docker-command=run\n  image=hadoop/docker-image\n
 cap-add=CHROOT,MKNOD",
@@ -773,7 +773,7 @@ namespace ContainerExecutor {
     ret = set_capabilities(&cmd_cfg, &container_cfg, &buff);
     ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
 
-    container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n";
+    container_executor_cfg_contents = "[docker]\n  docker.trusted.registries=hadoop\n";
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
     if (ret != 0) {
@@ -790,7 +790,7 @@ namespace ContainerExecutor {
     reset_args(&buff);
     int ret = 0;
     std::string container_executor_cfg_contents = "[docker]\n"
-        "  docker.privileged-containers.registries=hadoop\n"
+        "  docker.trusted.registries=hadoop\n"
         "  docker.allowed.devices=/dev/test-device,/dev/device2,regex:/dev/nvidia.*,regex:/dev/gpu-uvm.*";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
@@ -910,7 +910,7 @@ namespace ContainerExecutor {
     struct configuration container_cfg, cmd_cfg;
     struct args buff = ARGS_INITIAL_VALUE;
     int ret = 0;
-    std::string container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n
 "
+    std::string container_executor_cfg_contents = "[docker]\n  docker.trusted.registries=hadoop\n
 "
                                                               "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut\n
 "
                                                               "docker.allowed.ro-mounts=/etc/passwd";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
@@ -1037,7 +1037,7 @@ namespace ContainerExecutor {
     struct args buff = ARGS_INITIAL_VALUE;
     int ret = 0;
 
-    std::string container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n
 "
+    std::string container_executor_cfg_contents = "[docker]\n  docker.trusted.registries=hadoop\n
 "
                                                               "docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut\n
 "
                                                               "docker.allowed.ro-mounts=/etc/passwd,/etc/group";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
@@ -1118,7 +1118,7 @@ namespace ContainerExecutor {
       free(actual);
     }
 
-    container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n";
+    container_executor_cfg_contents = "[docker]\n  docker.trusted.registries=hadoop\n";
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
     if (ret != 0) {
@@ -1136,7 +1136,7 @@ namespace ContainerExecutor {
     std::string container_executor_contents = "[docker]\n  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
         "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n "
         "  docker.privileged-containers.enabled=1\n  docker.allowed.capabilities=CHOWN,SETUID\n"
-        "  docker.allowed.devices=/dev/test\n  docker.privileged-containers.registries=hadoop\n";
+        "  docker.allowed.devices=/dev/test\n  docker.trusted.registries=hadoop\n";
     write_file(container_executor_cfg_file, container_executor_contents);
     int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
     if (ret != 0) {
@@ -1180,7 +1180,7 @@ namespace ContainerExecutor {
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
             "  launch-command=bash,test_script.sh,arg1,arg2",
         "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm"
-            " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image"));
+            " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image
bash test_script.sh arg1 arg2"));
 
     // Test non-privileged container and drop all privileges
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
@@ -1202,7 +1202,7 @@ namespace ContainerExecutor {
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
             "  launch-command=bash,test_script.sh,arg1,arg2",
         "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge"
-            " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image"));
+            " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image
bash test_script.sh arg1 arg2"));
 
     // Test privileged container
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
@@ -1237,7 +1237,7 @@ namespace ContainerExecutor {
             "  launch-command=bash,test_script.sh,arg1,arg2",
         "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge
--cap-drop=ALL "
             "--hostname=host-id --group-add 1000 --group-add 1001 "
-            "docker-image"));
+            "docker-image bash test_script.sh arg1 arg2"));
 
     std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
 
@@ -1318,7 +1318,7 @@ namespace ContainerExecutor {
         "  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
         "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n "
         "  docker.privileged-containers.enabled=1\n  docker.allowed.capabilities=CHOWN,SETUID\n"
-        "  docker.allowed.devices=/dev/test\n  docker.privileged-containers.registries=hadoop\n";
+        "  docker.allowed.devices=/dev/test\n  docker.trusted.registries=hadoop\n";
     write_file(container_executor_cfg_file, container_executor_contents);
     int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
     if (ret != 0) {
@@ -1357,12 +1357,12 @@ namespace ContainerExecutor {
   TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
 
     std::string container_executor_contents[] = {"[docker]\n  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.allowed.rw-mounts=/tmp\n 
docker.allowed.networks=bridge\n"
                                                      "  docker.allowed.capabilities=CHOWN,SETUID\n"
                                                      "  docker.allowed.devices=/dev/test",
                                                  "[docker]\n  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.allowed.rw-mounts=/tmp\n 
docker.allowed.networks=bridge\n"
                                                      "  docker.allowed.capabilities=CHOWN,SETUID\n"
                                                      "  privileged=0\n"
@@ -1386,7 +1386,7 @@ namespace ContainerExecutor {
       file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
           "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n
 image=docker-image\n"
               "  user=nobody\n  launch-command=bash,test_script.sh,arg1,arg2",
-          "run --name=container_e1_12312_11111_02_000001 --user=nobody --cap-drop=ALL docker-image"));
+          "run --name=container_e1_12312_11111_02_000001 --user=nobody --cap-drop=ALL docker-image
bash test_script.sh arg1 arg2"));
 
       file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
           "[docker-command-execution]\n"
@@ -1407,7 +1407,7 @@ namespace ContainerExecutor {
               "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
               "  launch-command=bash,test_script.sh,arg1,arg2",
           "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm"
-              " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image"));
+              " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image
bash test_script.sh arg1 arg2"));
 
       file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
           "[docker-command-execution]\n"
@@ -1428,7 +1428,7 @@ namespace ContainerExecutor {
               "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
               "  launch-command=bash,test_script.sh,arg1,arg2",
           "run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge"
-              " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image"));
+              " --cgroup-parent=ctr-cgroup --cap-drop=ALL --hostname=host-id nothadoop/docker-image
bash test_script.sh arg1 arg2"));
 
       std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
       bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
@@ -1549,23 +1549,23 @@ namespace ContainerExecutor {
   TEST_F(TestDockerUtil, test_docker_no_new_privileges) {
 
     std::string container_executor_contents[] = {"[docker]\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.privileged-containers.enabled=false\n"
                                                      "  docker.no-new-privileges.enabled=true",
                                                  "[docker]\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.privileged-containers.enabled=true\n"
                                                      "  docker.no-new-privileges.enabled=true",
                                                  "[docker]\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.privileged-containers.enabled=true\n"
                                                      "  docker.no-new-privileges.enabled=true",
                                                  "[docker]\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.privileged-containers.enabled=false\n"
                                                      "  docker.no-new-privileges.enabled=false",
                                                  "[docker]\n"
-                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.trusted.registries=hadoop\n"
                                                      "  docker.privileged-containers.enabled=true\n"
                                                      "  docker.no-new-privileges.enabled=false"};
     for (int i = 0; i < 2; ++i) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 0f49a06..c6f965a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -206,7 +206,7 @@ are allowed. It contains the following properties:
 | `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to
mount in read-write mode. By default, no directories are allowed to mounted. |
 | `docker.host-pid-namespace.enabled` | Set to "true" or "false" to enable or disable using
the host's PID namespace. Default value is "false". |
 | `docker.privileged-containers.enabled` | Set to "true" or "false" to enable or disable
launching privileged containers. Default value is "false". |
-| `docker.privileged-containers.registries` | Comma separated list of trusted docker registries
for running trusted privileged docker containers.  By default, no registries are defined.
|
+| `docker.trusted.registries` | Comma separated list of trusted docker registries for running
trusted privileged docker containers.  By default, no registries are defined. |
 | `docker.inspect.max.retries` | Integer value to check docker container readiness.  Each
inspection is set with 3 seconds delay.  Default value of 10 will wait 30 seconds for docker
container to become ready before marked as container failed. |
 | `docker.no-new-privileges.enabled` | Enable/disable the no-new-privileges flag for docker
run. Set to "true" to enable, disabled by default. |
 
@@ -230,7 +230,7 @@ yarn.nodemanager.linux-container-executor.group=yarn
 [docker]
   module.enabled=true
   docker.privileged-containers.enabled=true
-  docker.privileged-containers.registries=centos
+  docker.trusted.registries=centos
   docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE
   docker.allowed.networks=bridge,host,none
   docker.allowed.ro-mounts=/sys/fs/cgroup
@@ -372,7 +372,7 @@ Privileged docker container can interact with host system devices.  This
can cau
 
 The default behavior is disallow any privileged docker containers.  When `docker.privileged-containers.enabled`
is set to enabled, docker image can run with root privileges in the docker container, but
access to host level devices are disabled.  This allows developer and tester to run docker
images from internet without causing harm to host operating system.
 
-When docker images have been certified by developers and testers to be trustworthy.  The
trusted image can be promoted to trusted docker registry.  System administrator can define
`docker.privileged-containers.registries`, and setup private docker registry server to promote
trusted images.
+When docker images have been certified by developers and testers to be trustworthy.  The
trusted image can be promoted to trusted docker registry.  System administrator can define
`docker.trusted.registries`, and setup private docker registry server to promote trusted images.
 
 Trusted images are allowed to mount external devices such as HDFS via NFS gateway, or host
level Hadoop configuration.  If system administrators allow writing to external volumes using
`docker.allow.rw-mounts directive`, privileged docker container can have full control of host
level files in the predefined volumes.
 
@@ -436,3 +436,30 @@ To run a Spark shell in Docker containers, run the following command:
 
 Note that the application master and executors are configured
 independently. In this example, we are using the hadoop-docker image for both.
+
+Docker Container ENTRYPOINT Support
+------------------------------------
+
+When Docker support was introduced to Hadoop 2.x, the platform was designed to
+run existing Hadoop programs inside Docker container.  Log redirection and
+environment setup are integrated with Node Manager.  In Hadoop 3.x, Hadoop
+Docker support extends beyond running Hadoop workload, and support Docker container
+in Docker native form using ENTRYPOINT from dockerfile.  Application can decide to
+support YARN mode as default or Docker mode as default by defining
+YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE environment variable.
+System administrator can also set as default setting for the cluster to make
+ENTRY_POINT as default mode of operation.
+
+In yarn-site.xml, add YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE to
+node manager environment white list:
+```
+<property>
+        <name>yarn.nodemanager.env-whitelist</name>
+        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME,YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE</value>
+</property>
+```
+
+In yarn-env.sh, define:
+```
+export YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE=true
+```

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index 4163635..03fec79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -20,7 +20,7 @@ This document describes some example service definitions (`Yarnfile`).
 
 ## Apache web server - httpd (with registry DNS)
 
-For this example to work, centos/httpd-24-centos7 image must be included in `docker.privileged-containers.registries`.
+For this example to work, centos/httpd-24-centos7 image must be included in `docker.trusted.registries`.
 For server side configuration, please refer to [Running Applications in Docker Containers](../DockerContainers.html)
document.
 
 Below is the `Yarnfile` for a service called `httpd-service` with two `httpd` instances.
@@ -163,3 +163,38 @@ where `service-name` is optional. If omitted, it uses the name defined
in the `Y
 
 Look up your IPs at the RM REST endpoint `http://<RM host>:8088/app/v1/services/httpd-service`.
 Then visit port 8080 for each IP to view the pages.
+
+## Docker image ENTRYPOINT support
+
+Docker images may have built with ENTRYPOINT to enable start up of docker image without any
parameters.
+When passing parameters to ENTRYPOINT enabled image, `launch_command` is delimited by comma
(,).
+
+{
+  "name": "sleeper-service",
+  "version": "1",
+  "components" :
+  [
+    {
+      "name": "sleeper",
+      "number_of_containers": 2,
+      "artifact": {
+        "id": "hadoop/centos:latest",
+        "type": "DOCKER"
+      },
+      "launch_command": "sleep,90000",
+      "resource": {
+        "cpus": 1,
+        "memory": "256"
+      },
+      "restart_policy": "ON_FAILURE",
+      "configuration": {
+        "env": {
+          "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE":"true"
+        },
+        "properties": {
+          "docker.network": "host"
+        }
+      }
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31998643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index f1dc81b..4bfa742 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -225,7 +225,7 @@ One or more components of the service. If the service is HBase say, then
the com
 |dependencies|An array of service components which should be in READY state (as defined by
readiness check), before this component can be started. The dependencies across all components
of a service should be represented as a DAG.|false|string array||
 |readiness_check|Readiness check for this component.|false|ReadinessCheck||
 |artifact|Artifact of the component (optional). If not specified, the service level global
artifact takes effect.|false|Artifact||
-|launch_command|The custom launch command of this component (optional for DOCKER component,
required otherwise). When specified at the component level, it overrides the value specified
at the global level (if any).|false|string||
+|launch_command|The custom launch command of this component (optional for DOCKER component,
required otherwise). When specified at the component level, it overrides the value specified
at the global level (if any). If docker image supports ENTRYPOINT, launch_command is delimited
by comma(,) instead of space.|false|string||
 |resource|Resource of this component (optional). If not specified, the service level global
resource takes effect.|false|Resource||
 |number_of_containers|Number of containers for this component (optional). If not specified,
the service level global number_of_containers takes effect.|false|integer (int64)||
 |containers|Containers of a started component. Specifying a value for this attribute for
the POST payload raises a validation error. This blob is available only in the GET response
of a started service.|false|Container array||


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message