hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [11/50] [abbrv] hadoop git commit: YARN-7717. Add configuration consistency for module.enabled and docker.privileged-containers.enabled. Contributed by Eric Badger.
Date Wed, 24 Jan 2018 22:35:40 GMT
YARN-7717. Add configuration consistency for module.enabled and docker.privileged-containers.enabled.
Contributed by Eric Badger.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a68e445d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a68e445d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a68e445d

Branch: refs/heads/HDFS-7240
Commit: a68e445dc682f4a123cdf016ce1aa46e550c7fdf
Parents: 6e42d05
Author: Miklos Szegedi <szegedim@apache.org>
Authored: Wed Jan 17 14:11:14 2018 -0800
Committer: Miklos Szegedi <szegedim@apache.org>
Committed: Wed Jan 17 14:11:14 2018 -0800

----------------------------------------------------------------------
 .../hadoop-yarn/conf/container-executor.cfg     |  4 +--
 .../impl/container-executor.c                   | 31 +++++++++++-------
 .../container-executor/impl/utils/docker-util.c |  6 ++--
 .../test/test-container-executor.c              |  6 ++++
 .../test/utils/test_docker_util.cc              | 34 ++++++++++++--------
 .../src/site/markdown/DockerContainers.md       |  4 +--
 6 files changed, 53 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68e445d/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index 4c698b5..36676b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -2,7 +2,7 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
 banned.users=#comma separated list of users who can not run applications
 min.user.id=1000#Prevent other super-users
 allowed.system.users=##comma separated list of system users who CAN run applications
-feature.tc.enabled=0
+feature.tc.enabled=false
 
 # The configs below deal with settings for Docker
 #[docker]
@@ -13,7 +13,7 @@ feature.tc.enabled=0
 #  docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
 #  docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
 #  docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write,
add the yarn local and log dirs to this list to run Hadoop jobs
-#  docker.privileged-containers.enabled=0
+#  docker.privileged-containers.enabled=false
 #  docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers
 
 # The configs below deal with settings for FPGA resource

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68e445d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 98e2d6e..b0b8e76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -39,6 +39,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <limits.h>
 #include <sys/stat.h>
 #include <sys/mount.h>
@@ -441,19 +442,25 @@ int is_feature_enabled(const char* feature_key, int default_value,
     int enabled = default_value;
 
     if (enabled_str != NULL) {
-        char *end_ptr = NULL;
-        enabled = strtol(enabled_str, &end_ptr, 10);
-
-        if ((enabled_str == end_ptr || *end_ptr != '\0') ||
-            (enabled < 0 || enabled > 1)) {
-              fprintf(LOGFILE, "Illegal value '%s' for '%s' in configuration. "
-              "Using default value: %d.\n", enabled_str, feature_key,
-              default_value);
-              fflush(LOGFILE);
-              free(enabled_str);
-              return default_value;
-        }
+        if (strcasecmp(enabled_str, "true") == 0) {
+            enabled = 1;
+        } else if (strcasecmp(enabled_str, "false") == 0) {
+            enabled = 0;
+        } else {
+            char *end_ptr = NULL;
+            enabled = strtol(enabled_str, &end_ptr, 10);
+
+            if ((enabled_str == end_ptr || *end_ptr != '\0') ||
+                (enabled < 0 || enabled > 1)) {
+                  fprintf(LOGFILE, "Illegal value '%s' for '%s' in configuration. "
+                  "Using default value: %d.\n", enabled_str, feature_key,
+                  default_value);
+                  fflush(LOGFILE);
+                  free(enabled_str);
+                  return default_value;
+            }
 
+        }
         free(enabled_str);
         return enabled;
     } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68e445d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index b5cb551..46b3e1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -17,6 +17,7 @@
  */
 
 #include <string.h>
+#include <strings.h>
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <ctype.h>
@@ -1045,9 +1046,10 @@ static int set_privileged(const struct configuration *command_config,
const stru
       = get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION,
conf);
   int ret = 0;
 
-  if (value != NULL && strcmp(value, "true") == 0) {
+  if (value != NULL && strcasecmp(value, "true") == 0 ) {
     if (privileged_container_enabled != NULL) {
-      if (strcmp(privileged_container_enabled, "1") == 0) {
+      if (strcmp(privileged_container_enabled, "1") == 0 ||
+          strcasecmp(privileged_container_enabled, "True") == 0) {
         ret = add_to_buffer(out, outlen, "--privileged ");
         if (ret != 0) {
           ret = BUFFER_TOO_SMALL;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68e445d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 8b0c0fa..3d32883 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -449,6 +449,8 @@ void test_is_feature_enabled() {
   fprintf(file, "feature.name4.enabled=asdkjfasdkljfklsdjf0\n");
   fprintf(file, "feature.name5.enabled=-1\n");
   fprintf(file, "feature.name6.enabled=2\n");
+  fprintf(file, "feature.name7.enabled=true\n");
+  fprintf(file, "feature.name8.enabled=True\n");
   fclose(file);
   read_config(filename, &exec_cfg);
   cfg = *(get_configuration_section("", &exec_cfg));
@@ -465,6 +467,10 @@ void test_is_feature_enabled() {
           enabled, &cfg);
   validate_feature_enabled_value(disabled, "feature.name6.enabled",
           disabled, &cfg);
+  validate_feature_enabled_value(enabled, "feature.name7.enabled",
+          disabled, &cfg);
+  validate_feature_enabled_value(enabled, "feature.name8.enabled",
+          disabled, &cfg);
 
 
   free_configuration(&exec_cfg);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68e445d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 416bf38..791b850 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -495,36 +495,42 @@ namespace ContainerExecutor {
     char buff[buff_len];
     int ret = 0;
     std::string container_executor_cfg_contents[] = {"[docker]\n  docker.privileged-containers.enabled=1",
+                                                     "[docker]\n  docker.privileged-containers.enabled=true",
+                                                     "[docker]\n  docker.privileged-containers.enabled=True",
                                                      "[docker]\n  docker.privileged-containers.enabled=0",
+                                                     "[docker]\n  docker.privileged-containers.enabled=false",
                                                      "[docker]\n"};
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+    std::vector<std::pair<std::string, std::string> >::const_iterator itr;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n  docker-command=run\n  privileged=true", "--privileged
"));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n  docker-command=run\n  privileged=false", ""));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n  docker-command=run", ""));
-    write_container_executor_cfg(container_executor_cfg_contents[0]);
-    ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+    for (int i = 0; i < 3; i++ ) {
+      write_container_executor_cfg(container_executor_cfg_contents[i]);
+      ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
 
-    std::vector<std::pair<std::string, std::string> >::const_iterator itr;
-    if (ret != 0) {
-      FAIL();
-    }
-    for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
-      memset(buff, 0, buff_len);
-      write_command_file(itr->first);
-      ret = read_config(docker_command_file.c_str(), &cmd_cfg);
       if (ret != 0) {
         FAIL();
       }
-      ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
-      ASSERT_EQ(0, ret);
-      ASSERT_STREQ(itr->second.c_str(), buff);
+      for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+        memset(buff, 0, buff_len);
+        write_command_file(itr->first);
+        ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+        if (ret != 0) {
+          FAIL();
+        }
+        ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+        ASSERT_EQ(0, ret);
+        ASSERT_STREQ(itr->second.c_str(), buff);
+      }
     }
 
+
     // check default case and when it's turned off
-    for (int i = 1; i < 3; ++i) {
+    for (int i = 3; i < 5; ++i) {
       write_container_executor_cfg(container_executor_cfg_contents[i]);
       ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
       if (ret != 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68e445d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index a3e4105..c83d0a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -196,7 +196,7 @@ are allowed. It contains the following properties:
 | `docker.allowed.networks` | Comma separated networks that containers are allowed to use.
If no network is specified when launching the container, the default Docker network will be
used. |
 | `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to
mount in read-only mode. By default, no directories are allowed to mounted. |
 | `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to
mount in read-write mode. By default, no directories are allowed to mounted. |
-| `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged
containers. Default value is 0. |
+| `docker.privileged-containers.enabled` | Set to "true" or "false" to enable or disable
launching privileged containers. Default value is "false". |
 
 Please note that if you wish to run Docker containers that require access to the YARN local
directories, you must add them to the docker.allowed.rw-mounts list.
 
@@ -209,7 +209,7 @@ The following properties are optional:
 | `min.user.id` | The minimum UID that is allowed to launch applications. The default is
no minimum |
 | `banned.users` | A comma-separated list of usernames who should not be allowed to launch
applications. The default setting is: yarn, mapred, hdfs, and bin. |
 | `allowed.system.users` | A comma-separated list of usernames who should be allowed to launch
applications even if their UIDs are below the configured minimum. If a user appears in allowed.system.users
and banned.users, the user will be considered banned. |
-| `feature.tc.enabled` | Must be 0 or 1. 0 means traffic control commands are disabled. 1
means traffic control commands are allowed. |
+| `feature.tc.enabled` | Must be "true" or "false". "false" means traffic control commands
are disabled. "true" means traffic control commands are allowed. |
 
 Part of a container-executor.cfg which allows Docker containers to be launched is below:
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message