hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: YARN-7286. Add support for docker to have no capabilities. Contributed by Eric Badger
Date Thu, 02 Nov 2017 14:38:28 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk cc3f3eca4 -> d00b6f7c1


YARN-7286. Add support for docker to have no capabilities. Contributed by Eric Badger


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d00b6f7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d00b6f7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d00b6f7c

Branch: refs/heads/trunk
Commit: d00b6f7c1ff2d7569ae9efdc6823ebcfb86ef2d4
Parents: cc3f3ec
Author: Jason Lowe <jlowe@apache.org>
Authored: Thu Nov 2 09:37:17 2017 -0500
Committer: Jason Lowe <jlowe@apache.org>
Committed: Thu Nov 2 09:37:17 2017 -0500

----------------------------------------------------------------------
 .../src/main/resources/yarn-default.xml         |  3 +-
 .../runtime/DockerLinuxContainerRuntime.java    | 30 ++++++++++++--
 .../runtime/TestDockerContainerRuntime.java     | 43 ++++++++++++++++++++
 3 files changed, 71 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8487e72..f4b2e61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1623,7 +1623,8 @@
     <description>This configuration setting determines the capabilities
       assigned to docker containers when they are launched. While these may not
       be case-sensitive from a docker perspective, it is best to keep these
-      uppercase.</description>
+      uppercase. To run without any capabilites, set this value to
+      "none" or "NONE"</description>
     <name>yarn.nodemanager.runtime.linux.docker.capabilities</name>
     <value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 6f7b6fd..a425cf8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -58,6 +58,7 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -187,6 +188,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime
{
   private boolean enableUserReMapping;
   private int userRemappingUidThreshold;
   private int userRemappingGidThreshold;
+  private Set<String> capabilities;
 
   /**
    * Return whether the given environment variables indicate that the operation
@@ -285,6 +287,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime
{
     userRemappingGidThreshold = conf.getInt(
       YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
       YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
+
+    capabilities = getDockerCapabilitiesFromConf();
+  }
+
+  private Set<String> getDockerCapabilitiesFromConf() throws
+      ContainerExecutionException {
+    Set<String> caps = new HashSet<>(Arrays.asList(
+        conf.getTrimmedStrings(
+        YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+    if(caps.contains("none") || caps.contains("NONE")) {
+      if(caps.size() > 1) {
+        String msg = "Mixing capabilities with the none keyword is" +
+            " not supported";
+        throw new ContainerExecutionException(msg);
+      }
+      caps = Collections.emptySet();
+    }
+
+    return caps;
+  }
+
+  public Set<String> getCapabilities() {
+    return capabilities;
   }
 
   @Override
@@ -602,10 +628,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime
{
         LOCALIZED_RESOURCES);
     @SuppressWarnings("unchecked")
     List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
-    Set<String> capabilities = new HashSet<>(Arrays.asList(
-        conf.getTrimmedStrings(
-            YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
-            YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
 
     @SuppressWarnings("unchecked")
     DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index b5a6497..76aca04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -65,6 +65,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -1304,4 +1305,46 @@ public class TestDockerContainerRuntime {
     Assert.assertEquals("  workdir=/test_container_work_dir",
         dockerCommands.get(counter++));
   }
+
+  @Test
+  public void testDockerCapabilities()
+      throws ContainerExecutionException, PrivilegedOperationException,
+      IOException {
+    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+        mockExecutor, mockCGroupsHandler);
+    try {
+      conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+          "none", "CHOWN", "DAC_OVERRIDE");
+      runtime.initialize(conf, null);
+      Assert.fail("Initialize didn't fail with invalid capabilities " +
+          "'none', 'CHOWN', 'DAC_OVERRIDE'");
+    } catch (ContainerExecutionException e) {
+    }
+
+    try {
+      conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+          "CHOWN", "DAC_OVERRIDE", "NONE");
+      runtime.initialize(conf, null);
+      Assert.fail("Initialize didn't fail with invalid capabilities " +
+          "'CHOWN', 'DAC_OVERRIDE', 'NONE'");
+    } catch (ContainerExecutionException e) {
+    }
+
+    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        "NONE");
+    runtime.initialize(conf, null);
+    Assert.assertEquals(0, runtime.getCapabilities().size());
+
+    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        "none");
+    runtime.initialize(conf, null);
+    Assert.assertEquals(0, runtime.getCapabilities().size());
+
+    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        "CHOWN", "DAC_OVERRIDE");
+    runtime.initialize(conf, null);
+    Iterator<String> it = runtime.getCapabilities().iterator();
+    Assert.assertEquals("CHOWN", it.next());
+    Assert.assertEquals("DAC_OVERRIDE", it.next());
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message