hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject hadoop git commit: YARN-5388. Deprecate and remove DockerContainerExecutor. (Daniel Templeton via kasha)
Date Tue, 25 Oct 2016 22:26:37 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 aedd5c4c1 -> e4ba92abe


YARN-5388. Deprecate and remove DockerContainerExecutor. (Daniel Templeton via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4ba92ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4ba92ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4ba92ab

Branch: refs/heads/branch-2
Commit: e4ba92abeecfd689bdc46b35abbd1da68520b6b0
Parents: aedd5c4
Author: Karthik Kambatla <kasha@cloudera.com>
Authored: Tue Oct 25 15:26:27 2016 -0700
Committer: Karthik Kambatla <kasha@cloudera.com>
Committed: Tue Oct 25 15:26:27 2016 -0700

----------------------------------------------------------------------
 .../nodemanager/DockerContainerExecutor.java    | 49 +++++++++++---------
 .../site/markdown/DockerContainerExecutor.md.vm |  9 ++++
 2 files changed, 37 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ba92ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index 1390214..b2f746e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -74,7 +74,13 @@ import com.google.common.base.Strings;
  * currently only supports simple authentication mode. It shares a lot of code
  * with the DefaultContainerExecutor (and it may make sense to pull out those
  * common pieces later).
+ * @deprecated The {@code DockerContainerExecutor} class has several
+ *  limitations, not the least of which is that if used, <b>all</b> containers
+ *  are launched in Docker containers. The {@link LinuxContainerExecutor}
+ *  supports selectively launching containers in Docker containers and should
+ *  be used instead.
  */
+@Deprecated
 public class DockerContainerExecutor extends ContainerExecutor {
   private static final Log LOG = LogFactory
     .getLog(DockerContainerExecutor.class);
@@ -324,11 +330,12 @@ public class DockerContainerExecutor extends ContainerExecutor {
     return 0;
   }
 
-  @Override
   /**
    * Filter the environment variables that may conflict with the ones set in
    * the docker image and write them out to an OutputStream.
+   * @throws IOException if there's an issue writing out the launch file.
    */
+  @Override
   public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
       Map<Path, List<String>> resources, List<String> command, Path logDir,
       String user) throws IOException {
@@ -336,7 +343,7 @@ public class DockerContainerExecutor extends ContainerExecutor {
       ContainerLaunch.ShellScriptBuilder.create();
 
     //Remove environments that may conflict with the ones in Docker image.
-    Set<String> exclusionSet = new HashSet<String>();
+    Set<String> exclusionSet = new HashSet<>();
     exclusionSet.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
     exclusionSet.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name());
     exclusionSet.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name());
@@ -347,14 +354,23 @@ public class DockerContainerExecutor extends ContainerExecutor {
     if (environment != null) {
       for (Map.Entry<String,String> env : environment.entrySet()) {
         if (!exclusionSet.contains(env.getKey())) {
-          sb.env(env.getKey().toString(), env.getValue().toString());
+          sb.env(env.getKey(), env.getValue());
         }
       }
     }
     if (resources != null) {
       for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
         for (String linkName : entry.getValue()) {
-          sb.symlink(entry.getKey(), new Path(linkName));
+          if (new Path(linkName).getName().equals(WILDCARD)) {
+            // If this is a wildcarded path, link to everything in the
+            // directory from the working directory
+            for (File wildLink : readDirAsUser(user, entry.getKey())) {
+              sb.symlink(new Path(wildLink.toString()),
+                  new Path(wildLink.getName()));
+            }
+          } else {
+            sb.symlink(entry.getKey(), new Path(linkName));
+          }
         }
       }
     }
@@ -370,26 +386,17 @@ public class DockerContainerExecutor extends ContainerExecutor {
 
     sb.command(command);
 
-    PrintStream pout = null;
-    PrintStream ps = null;
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    try {
-      pout = new PrintStream(out, false, "UTF-8");
-      if (LOG.isDebugEnabled()) {
-        ps = new PrintStream(baos, false, "UTF-8");
-        sb.write(ps);
-      }
+    try (PrintStream pout = new PrintStream(out, false, "UTF-8")) {
       sb.write(pout);
-
-    } finally {
-      if (out != null) {
-        out.close();
-      }
-      if (ps != null) {
-        ps.close();
-      }
     }
+
     if (LOG.isDebugEnabled()) {
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+      try (PrintStream ps = new PrintStream(baos, false, "UTF-8")) {
+        sb.write(ps);
+      }
+
       LOG.debug("Script: " + baos.toString("UTF-8"));
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ba92ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
index fbfe04b..1fa94ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
@@ -22,6 +22,15 @@ Docker Container Executor
 * [Docker Image Requirements](#Docker_Image_Requirements)
 * [Working example of yarn launched docker containers](#Working_example_of_yarn_launched_docker_containers)
 
+DEPRECATED
+----------
+
+The Docker Container Executor is not a recommended solution for providing Docker
+support in a YARN cluster.  The Linux Container Executor should be used instead.
+The documentation on [using CGroups](NodeManagerCgroups.html) and
+[secure containers](SecureContainer.html) contain basic information about
+configuring the Linux Container Executor.
+
 Overview
 --------
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message