hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject [05/38] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.
Date Tue, 04 Aug 2015 00:12:46 GMT
YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4c774a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4c774a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4c774a5

Branch: refs/heads/YARN-2928
Commit: d4c774a5399d1f776e1cb06bebb7189c24b9a69b
Parents: 5f29c6b
Author: Varun Vasudev <vvasudev@apache.org>
Authored: Mon Jul 27 11:57:40 2015 -0700
Committer: Zhijie Shen <zjshen@apache.org>
Committed: Mon Aug 3 17:02:06 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |   4 +
 .../server/nodemanager/ContainerExecutor.java   |  23 +-
 .../nodemanager/DefaultContainerExecutor.java   |   2 +-
 .../nodemanager/DockerContainerExecutor.java    |   2 +-
 .../nodemanager/LinuxContainerExecutor.java     | 222 +++++++--------
 .../launcher/ContainerLaunch.java               |  15 +
 .../linux/privileged/PrivilegedOperation.java   |  46 +++-
 .../PrivilegedOperationException.java           |  30 +-
 .../privileged/PrivilegedOperationExecutor.java |  30 +-
 .../linux/resources/CGroupsHandler.java         |   8 +
 .../linux/resources/CGroupsHandlerImpl.java     |  12 +-
 .../runtime/DefaultLinuxContainerRuntime.java   | 148 ++++++++++
 .../DelegatingLinuxContainerRuntime.java        | 110 ++++++++
 .../runtime/DockerLinuxContainerRuntime.java    | 273 +++++++++++++++++++
 .../linux/runtime/LinuxContainerRuntime.java    |  38 +++
 .../runtime/LinuxContainerRuntimeConstants.java |  69 +++++
 .../linux/runtime/docker/DockerClient.java      |  82 ++++++
 .../linux/runtime/docker/DockerCommand.java     |  66 +++++
 .../linux/runtime/docker/DockerLoadCommand.java |  30 ++
 .../linux/runtime/docker/DockerRunCommand.java  | 107 ++++++++
 .../runtime/ContainerExecutionException.java    |  85 ++++++
 .../runtime/ContainerRuntime.java               |  50 ++++
 .../runtime/ContainerRuntimeConstants.java      |  33 +++
 .../runtime/ContainerRuntimeContext.java        | 105 +++++++
 .../executor/ContainerLivenessContext.java      |  13 +
 .../executor/ContainerReacquisitionContext.java |  13 +
 .../executor/ContainerSignalContext.java        |  13 +
 .../executor/ContainerStartContext.java         |  23 +-
 .../TestLinuxContainerExecutorWithMocks.java    | 118 +++++---
 .../TestPrivilegedOperationExecutor.java        |   8 +-
 .../runtime/TestDockerContainerRuntime.java     | 219 +++++++++++++++
 31 files changed, 1815 insertions(+), 182 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0b7d544..690dea4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -274,6 +274,10 @@ Release 2.8.0 - UNRELEASED
     YARN-3852. Add docker container support to container-executor
     (Abin Shahab via vvasudev)
 
+    YARN-3853. Add docker container runtime support to LinuxContainterExecutor.
+    (Sidharta Seethana via vvasudev)
+
+
   IMPROVEMENTS
 
     YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 79f9b0d..68bfbbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -24,8 +24,10 @@ import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -39,6 +41,7 @@ import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -160,7 +163,7 @@ public abstract class ContainerExecutor implements Configurable {
    * @return true if container is still alive
    * @throws IOException
    */
-  public abstract boolean isContainerProcessAlive(ContainerLivenessContext ctx)
+  public abstract boolean isContainerAlive(ContainerLivenessContext ctx)
       throws IOException;
 
   /**
@@ -174,6 +177,7 @@ public abstract class ContainerExecutor implements Configurable {
    */
   public int reacquireContainer(ContainerReacquisitionContext ctx)
       throws IOException, InterruptedException {
+    Container container = ctx.getContainer();
     String user = ctx.getUser();
     ContainerId containerId = ctx.getContainerId();
 
@@ -193,10 +197,11 @@ public abstract class ContainerExecutor implements Configurable {
     LOG.info("Reacquiring " + containerId + " with pid " + pid);
     ContainerLivenessContext livenessContext = new ContainerLivenessContext
         .Builder()
+        .setContainer(container)
         .setUser(user)
         .setPid(pid)
         .build();
-    while(isContainerProcessAlive(livenessContext)) {
+    while(isContainerAlive(livenessContext)) {
       Thread.sleep(1000);
     }
 
@@ -243,9 +248,20 @@ public abstract class ContainerExecutor implements Configurable {
     Map<Path, List<String>> resources, List<String> command) throws IOException{
     ContainerLaunch.ShellScriptBuilder sb =
       ContainerLaunch.ShellScriptBuilder.create();
+    Set<String> whitelist = new HashSet<String>();
+    whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
+    whitelist.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name());
+    whitelist.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name());
+    whitelist.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name());
+    whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
+    whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
     if (environment != null) {
       for (Map.Entry<String,String> env : environment.entrySet()) {
-        sb.env(env.getKey().toString(), env.getValue().toString());
+        if (!whitelist.contains(env.getKey())) {
+          sb.env(env.getKey().toString(), env.getValue().toString());
+        } else {
+          sb.whitelistedEnv(env.getKey().toString(), env.getValue().toString());
+        }
       }
     }
     if (resources != null) {
@@ -492,6 +508,7 @@ public abstract class ContainerExecutor implements Configurable {
       try {
         Thread.sleep(delay);
         containerExecutor.signalContainer(new ContainerSignalContext.Builder()
+            .setContainer(container)
             .setUser(user)
             .setPid(pid)
             .setSignal(signal)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index b9be2b1..5819f23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -430,7 +430,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
   }
 
   @Override
-  public boolean isContainerProcessAlive(ContainerLivenessContext ctx)
+  public boolean isContainerAlive(ContainerLivenessContext ctx)
       throws IOException {
     String pid = ctx.getPid();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index d3b5d0a..9dffff3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -413,7 +413,7 @@ public class DockerContainerExecutor extends ContainerExecutor {
   }
 
   @Override
-  public boolean isContainerProcessAlive(ContainerLivenessContext ctx)
+  public boolean isContainerAlive(ContainerLivenessContext ctx)
     throws IOException {
     String pid = ctx.getPid();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index b936969..0670d95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -20,15 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Pattern;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -46,10 +37,14 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DelegatingLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
@@ -60,6 +55,22 @@ import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler
 import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+
+/** Container execution for Linux. Provides linux-specific localization
+ * mechanisms, resource management via cgroups and can switch between multiple
+ * container runtimes - e.g Standard "Process Tree", Docker etc
+ */
+
 public class LinuxContainerExecutor extends ContainerExecutor {
 
   private static final Log LOG = LogFactory
@@ -73,6 +84,15 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   private int containerSchedPriorityAdjustment = 0;
   private boolean containerLimitUsers;
   private ResourceHandler resourceHandlerChain;
+  private LinuxContainerRuntime linuxContainerRuntime;
+
+  public LinuxContainerExecutor() {
+  }
+
+  // created primarily for testing
+  public LinuxContainerExecutor(LinuxContainerRuntime linuxContainerRuntime) {
+    this.linuxContainerRuntime = linuxContainerRuntime;
+  }
 
   @Override
   public void setConf(Configuration conf) {
@@ -85,10 +105,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     resourcesHandler.setConf(conf);
 
     if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) {
-      containerSchedPriorityIsSet = true;
-      containerSchedPriorityAdjustment = conf
-          .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 
-          YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
+     containerSchedPriorityIsSet = true;
+     containerSchedPriorityAdjustment = conf
+         .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
+             YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
     }
     nonsecureLocalUser = conf.get(
         YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY,
@@ -122,46 +142,6 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     }
   }
 
-  /**
-   * List of commands that the setuid script will execute.
-   */
-  enum Commands {
-    INITIALIZE_CONTAINER(0),
-    LAUNCH_CONTAINER(1),
-    SIGNAL_CONTAINER(2),
-    DELETE_AS_USER(3);
-
-    private int value;
-    Commands(int value) {
-      this.value = value;
-    }
-    int getValue() {
-      return value;
-    }
-  }
-
-  /**
-   * Result codes returned from the C container-executor.
-   * These must match the values in container-executor.h.
-   */
-  enum ResultCode {
-    OK(0),
-    INVALID_USER_NAME(2),
-    UNABLE_TO_EXECUTE_CONTAINER_SCRIPT(7),
-    INVALID_CONTAINER_PID(9),
-    INVALID_CONTAINER_EXEC_PERMISSIONS(22),
-    INVALID_CONFIG_FILE(24),
-    WRITE_CGROUP_FAILED(27);
-
-    private final int value;
-    ResultCode(int value) {
-      this.value = value;
-    }
-    int getValue() {
-      return value;
-    }
-  }
-
   protected String getContainerExecutorExecutablePath(Configuration conf) {
     String yarnHomeEnvVar =
         System.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
@@ -203,9 +183,9 @@ public class LinuxContainerExecutor extends ContainerExecutor {
           + " (error=" + exitCode + ")", e);
     }
 
-    try {
-      Configuration conf = super.getConf();
+    Configuration conf = super.getConf();
 
+    try {
       resourceHandlerChain = ResourceHandlerModule
           .getConfiguredResourceHandlerChain(conf);
       if (resourceHandlerChain != null) {
@@ -216,9 +196,20 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       throw new IOException("Failed to bootstrap configured resource subsystems!");
     }
 
+    try {
+      if (linuxContainerRuntime == null) {
+        LinuxContainerRuntime runtime = new DelegatingLinuxContainerRuntime();
+
+        runtime.initialize(conf);
+        this.linuxContainerRuntime = runtime;
+      }
+    } catch (ContainerExecutionException e) {
+     throw new IOException("Failed to initialize linux container runtime(s)!");
+    }
+
     resourcesHandler.init(this);
   }
-  
+
   @Override
   public void startLocalizer(LocalizerStartContext ctx)
       throws IOException, InterruptedException {
@@ -238,7 +229,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     command.addAll(Arrays.asList(containerExecutorExe,
                    runAsUser,
                    user, 
-                   Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
+                   Integer.toString(PrivilegedOperation.RunAsUserCommand.INITIALIZE_CONTAINER.getValue()),
                    appId,
                    nmPrivateContainerTokensPath.toUri().getPath().toString(),
                    StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
@@ -294,6 +285,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     Path containerWorkDir = ctx.getContainerWorkDir();
     List<String> localDirs = ctx.getLocalDirs();
     List<String> logDirs = ctx.getLogDirs();
+    Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
 
     verifyUsernamePattern(user);
     String runAsUser = getRunAsUser(user);
@@ -351,50 +343,48 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       throw new IOException("ResourceHandlerChain.preStart() failed!");
     }
 
-    ShellCommandExecutor shExec = null;
-
     try {
       Path pidFilePath = getPidFilePath(containerId);
       if (pidFilePath != null) {
-        List<String> command = new ArrayList<String>();
-        addSchedPriorityCommand(command);
-        command.addAll(Arrays.asList(
-            containerExecutorExe, runAsUser, user, Integer
-                .toString(Commands.LAUNCH_CONTAINER.getValue()), appId,
-            containerIdStr, containerWorkDir.toString(),
-            nmPrivateContainerScriptPath.toUri().getPath().toString(),
-            nmPrivateTokensPath.toUri().getPath().toString(),
-            pidFilePath.toString(),
-            StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-                localDirs),
-            StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-                logDirs),
-            resourcesOptions));
+        List<String> prefixCommands= new ArrayList<>();
+        ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext
+            .Builder(container);
+
+        addSchedPriorityCommand(prefixCommands);
+        if (prefixCommands.size() > 0) {
+          builder.setExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS,
+              prefixCommands);
+        }
+
+        builder.setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources)
+            .setExecutionAttribute(RUN_AS_USER, runAsUser)
+            .setExecutionAttribute(USER, user)
+            .setExecutionAttribute(APPID, appId)
+            .setExecutionAttribute(CONTAINER_ID_STR, containerIdStr)
+            .setExecutionAttribute(CONTAINER_WORK_DIR, containerWorkDir)
+            .setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH,
+                nmPrivateContainerScriptPath)
+            .setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, nmPrivateTokensPath)
+            .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
+            .setExecutionAttribute(LOCAL_DIRS, localDirs)
+            .setExecutionAttribute(LOG_DIRS, logDirs)
+            .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
 
         if (tcCommandFile != null) {
-            command.add(tcCommandFile);
+          builder.setExecutionAttribute(TC_COMMAND_FILE, tcCommandFile);
         }
 
-        String[] commandArray = command.toArray(new String[command.size()]);
-        shExec = new ShellCommandExecutor(commandArray, null, // NM's cwd
-            container.getLaunchContext().getEnvironment()); // sanitized env
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("launchContainer: " + Arrays.toString(commandArray));
-        }
-        shExec.execute();
-        if (LOG.isDebugEnabled()) {
-          logOutput(shExec.getOutput());
-        }
+        linuxContainerRuntime.launchContainer(builder.build());
       } else {
         LOG.info("Container was marked as inactive. Returning terminated error");
         return ExitCode.TERMINATED.getExitCode();
       }
-    } catch (ExitCodeException e) {
-      int exitCode = shExec.getExitCode();
+    } catch (ContainerExecutionException e) {
+      int exitCode = e.getExitCode();
       LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
       // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
       // terminated/killed forcefully. In all other cases, log the
-      // container-executor's output
+      // output
       if (exitCode != ExitCode.FORCE_KILLED.getExitCode()
           && exitCode != ExitCode.TERMINATED.getExitCode()) {
         LOG.warn("Exception from container-launch with container ID: "
@@ -404,13 +394,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
         builder.append("Exception from container-launch.\n");
         builder.append("Container id: " + containerId + "\n");
         builder.append("Exit code: " + exitCode + "\n");
-        if (!Optional.fromNullable(e.getMessage()).or("").isEmpty()) {
-          builder.append("Exception message: " + e.getMessage() + "\n");
+        if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
+          builder.append("Exception message: " + e.getErrorOutput() + "\n");
         }
         builder.append("Stack trace: "
             + StringUtils.stringifyException(e) + "\n");
-        if (!shExec.getOutput().isEmpty()) {
-          builder.append("Shell output: " + shExec.getOutput() + "\n");
+        if (!e.getOutput().isEmpty()) {
+          builder.append("Shell output: " + e.getOutput() + "\n");
         }
         String diagnostics = builder.toString();
         logOutput(diagnostics);
@@ -433,10 +423,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
             "containerId: " + containerId + ". Exception: " + e);
       }
     }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Output from LinuxContainerExecutor's launchContainer follows:");
-      logOutput(shExec.getOutput());
-    }
+
     return 0;
   }
 
@@ -474,6 +461,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   @Override
   public boolean signalContainer(ContainerSignalContext ctx)
       throws IOException {
+    Container container = ctx.getContainer();
     String user = ctx.getUser();
     String pid = ctx.getPid();
     Signal signal = ctx.getSignal();
@@ -481,30 +469,27 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     verifyUsernamePattern(user);
     String runAsUser = getRunAsUser(user);
 
-    String[] command =
-        new String[] { containerExecutorExe,
-                   runAsUser,
-                   user,
-                   Integer.toString(Commands.SIGNAL_CONTAINER.getValue()),
-                   pid,
-                   Integer.toString(signal.getValue()) };
-    ShellCommandExecutor shExec = new ShellCommandExecutor(command);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("signalContainer: " + Arrays.toString(command));
-    }
+    ContainerRuntimeContext runtimeContext = new ContainerRuntimeContext
+        .Builder(container)
+        .setExecutionAttribute(RUN_AS_USER, runAsUser)
+        .setExecutionAttribute(USER, user)
+        .setExecutionAttribute(PID, pid)
+        .setExecutionAttribute(SIGNAL, signal)
+        .build();
+
     try {
-      shExec.execute();
-    } catch (ExitCodeException e) {
-      int ret_code = shExec.getExitCode();
-      if (ret_code == ResultCode.INVALID_CONTAINER_PID.getValue()) {
+      linuxContainerRuntime.signalContainer(runtimeContext);
+    } catch (ContainerExecutionException e) {
+      int retCode = e.getExitCode();
+      if (retCode == PrivilegedOperation.ResultCode.INVALID_CONTAINER_PID.getValue()) {
         return false;
       }
       LOG.warn("Error in signalling container " + pid + " with " + signal
-          + "; exit = " + ret_code, e);
-      logOutput(shExec.getOutput());
+          + "; exit = " + retCode, e);
+      logOutput(e.getOutput());
       throw new IOException("Problem signalling container " + pid + " with "
-          + signal + "; output: " + shExec.getOutput() + " and exitCode: "
-          + ret_code, e);
+          + signal + "; output: " + e.getOutput() + " and exitCode: "
+          + retCode, e);
     }
     return true;
   }
@@ -524,7 +509,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
         Arrays.asList(containerExecutorExe,
                     runAsUser,
                     user,
-                    Integer.toString(Commands.DELETE_AS_USER.getValue()),
+                    Integer.toString(PrivilegedOperation.
+                        RunAsUserCommand.DELETE_AS_USER.getValue()),
                     dirString));
     List<String> pathsToDelete = new ArrayList<String>();
     if (baseDirs == null || baseDirs.size() == 0) {
@@ -558,13 +544,15 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   }
   
   @Override
-  public boolean isContainerProcessAlive(ContainerLivenessContext ctx)
+  public boolean isContainerAlive(ContainerLivenessContext ctx)
       throws IOException {
     String user = ctx.getUser();
     String pid = ctx.getPid();
+    Container container = ctx.getContainer();
 
     // Send a test signal to the process as the user to see if it's alive
     return signalContainer(new ContainerSignalContext.Builder()
+        .setContainer(container)
         .setUser(user)
         .setPid(pid)
         .setSignal(Signal.NULL)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index af168c5..bf00d74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -303,6 +303,7 @@ public class ContainerLaunch implements Callable<Integer> {
         exec.activateContainer(containerID, pidFilePath);
         ret = exec.launchContainer(new ContainerStartContext.Builder()
             .setContainer(container)
+            .setLocalizedResources(localResources)
             .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath)
             .setNmPrivateTokensPath(nmPrivateTokensPath)
             .setUser(user)
@@ -427,6 +428,7 @@ public class ContainerLaunch implements Callable<Integer> {
 
         boolean result = exec.signalContainer(
             new ContainerSignalContext.Builder()
+                .setContainer(container)
                 .setUser(user)
                 .setPid(processId)
                 .setSignal(signal)
@@ -528,6 +530,8 @@ public class ContainerLaunch implements Callable<Integer> {
 
     public abstract void command(List<String> command) throws IOException;
 
+    public abstract void whitelistedEnv(String key, String value) throws IOException;
+
     public abstract void env(String key, String value) throws IOException;
 
     public final void symlink(Path src, Path dst) throws IOException {
@@ -586,6 +590,11 @@ public class ContainerLaunch implements Callable<Integer> {
     }
 
     @Override
+    public void whitelistedEnv(String key, String value) {
+      line("export ", key, "=${", key, ":-", "\"", value, "\"}");
+    }
+
+    @Override
     public void env(String key, String value) {
       line("export ", key, "=\"", value, "\"");
     }
@@ -627,6 +636,12 @@ public class ContainerLaunch implements Callable<Integer> {
     }
 
     @Override
+    public void whitelistedEnv(String key, String value) throws IOException {
+      lineWithLenCheck("@set ", key, "=", value);
+      errorCheck();
+    }
+
+    @Override
     public void env(String key, String value) throws IOException {
       lineWithLenCheck("@set ", key, "=", value);
       errorCheck();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
index f220cbd..cbbf7a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
@@ -45,10 +45,12 @@ public class PrivilegedOperation {
     LAUNCH_CONTAINER(""), //no CLI switch supported yet
     SIGNAL_CONTAINER(""), //no CLI switch supported yet
     DELETE_AS_USER(""), //no CLI switch supported yet
+    LAUNCH_DOCKER_CONTAINER(""), //no CLI switch supported yet
     TC_MODIFY_STATE("--tc-modify-state"),
     TC_READ_STATE("--tc-read-state"),
     TC_READ_STATS("--tc-read-stats"),
-    ADD_PID_TO_CGROUP(""); //no CLI switch supported yet.
+    ADD_PID_TO_CGROUP(""), //no CLI switch supported yet.
+    RUN_DOCKER_CMD("--run-docker");
 
     private final String option;
 
@@ -62,6 +64,7 @@ public class PrivilegedOperation {
   }
 
   public static final String CGROUP_ARG_PREFIX = "cgroups=";
+  public static final String CGROUP_ARG_NO_TASKS = "none";
 
   private final OperationType opType;
   private final List<String> args;
@@ -117,4 +120,45 @@ public class PrivilegedOperation {
   public int hashCode() {
     return opType.hashCode() + 97 * args.hashCode();
   }
+
+  /**
+   * List of commands that the container-executor will execute.
+   */
+  public enum RunAsUserCommand {
+    INITIALIZE_CONTAINER(0),
+    LAUNCH_CONTAINER(1),
+    SIGNAL_CONTAINER(2),
+    DELETE_AS_USER(3),
+    LAUNCH_DOCKER_CONTAINER(4);
+
+    private int value;
+    RunAsUserCommand(int value) {
+      this.value = value;
+    }
+    public int getValue() {
+      return value;
+    }
+  }
+
+  /**
+   * Result codes returned from the C container-executor.
+   * These must match the values in container-executor.h.
+   */
+  public enum ResultCode {
+    OK(0),
+    INVALID_USER_NAME(2),
+    UNABLE_TO_EXECUTE_CONTAINER_SCRIPT(7),
+    INVALID_CONTAINER_PID(9),
+    INVALID_CONTAINER_EXEC_PERMISSIONS(22),
+    INVALID_CONFIG_FILE(24),
+    WRITE_CGROUP_FAILED(27);
+
+    private final int value;
+    ResultCode(int value) {
+      this.value = value;
+    }
+    public int getValue() {
+      return value;
+    }
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java
index 20c234d..3622489 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 
 public class PrivilegedOperationException extends YarnException {
   private static final long serialVersionUID = 1L;
+  private Integer exitCode;
+  private String output;
+  private String errorOutput;
 
   public PrivilegedOperationException() {
     super();
@@ -33,11 +36,36 @@ public class PrivilegedOperationException extends YarnException {
     super(message);
   }
 
+  public PrivilegedOperationException(String message, Integer exitCode,
+      String output, String errorOutput) {
+    super(message);
+    this.exitCode = exitCode;
+    this.output = output;
+    this.errorOutput = errorOutput;
+  }
+
   public PrivilegedOperationException(Throwable cause) {
     super(cause);
   }
 
+  public PrivilegedOperationException(Throwable cause, Integer exitCode, String
+      output, String errorOutput) {
+    super(cause);
+    this.exitCode = exitCode;
+    this.output = output;
+    this.errorOutput = errorOutput;
+  }
   public PrivilegedOperationException(String message, Throwable cause) {
     super(message, cause);
   }
-}
+
+  public Integer getExitCode() {
+    return exitCode;
+  }
+
+  public String getOutput() {
+    return output;
+  }
+
+  public String getErrorOutput() { return errorOutput; }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
index 6fe0f5c..1d71938 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
@@ -20,6 +20,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -101,7 +102,13 @@ public class PrivilegedOperationExecutor {
     }
 
     fullCommand.add(containerExecutorExe);
-    fullCommand.add(operation.getOperationType().getOption());
+
+    String cliSwitch = operation.getOperationType().getOption();
+
+    if (!cliSwitch.isEmpty()) {
+      fullCommand.add(cliSwitch);
+    }
+
     fullCommand.addAll(operation.getArguments());
 
     String[] fullCommandArray =
@@ -142,6 +149,8 @@ public class PrivilegedOperationExecutor {
     try {
       exec.execute();
       if (LOG.isDebugEnabled()) {
+        LOG.debug("command array:");
+        LOG.debug(Arrays.toString(fullCommandArray));
         LOG.debug("Privileged Execution Operation Output:");
         LOG.debug(exec.getOutput());
       }
@@ -152,7 +161,11 @@ public class PrivilegedOperationExecutor {
           .append(System.lineSeparator()).append(exec.getOutput()).toString();
 
       LOG.warn(logLine);
-      throw new PrivilegedOperationException(e);
+
+      //stderr from shell executor seems to be stuffed into the exception
+      //'message' - so, we have to extract it and set it as the error out
+      throw new PrivilegedOperationException(e, e.getExitCode(),
+          exec.getOutput(), e.getMessage());
     } catch (IOException e) {
       LOG.warn("IOException executing command: ", e);
       throw new PrivilegedOperationException(e);
@@ -202,7 +215,7 @@ public class PrivilegedOperationExecutor {
 
     StringBuffer finalOpArg = new StringBuffer(PrivilegedOperation
         .CGROUP_ARG_PREFIX);
-    boolean noneArgsOnly = true;
+    boolean noTasks = true;
 
     for (PrivilegedOperation op : ops) {
       if (!op.getOperationType()
@@ -227,23 +240,24 @@ public class PrivilegedOperationExecutor {
         throw new PrivilegedOperationException("Invalid argument: " + arg);
       }
 
-      if (tasksFile.equals("none")) {
+      if (tasksFile.equals(PrivilegedOperation.CGROUP_ARG_NO_TASKS)) {
         //Don't append to finalOpArg
         continue;
       }
 
-      if (noneArgsOnly == false) {
+      if (noTasks == false) {
         //We have already appended at least one tasks file.
         finalOpArg.append(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR);
         finalOpArg.append(tasksFile);
       } else {
         finalOpArg.append(tasksFile);
-        noneArgsOnly = false;
+        noTasks = false;
       }
     }
 
-    if (noneArgsOnly) {
-      finalOpArg.append("none"); //there were no tasks file to append
+    if (noTasks) {
+      finalOpArg.append(PrivilegedOperation.CGROUP_ARG_NO_TASKS); //there
+      // were no tasks file to append
     }
 
     PrivilegedOperation finalOp = new PrivilegedOperation(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
index 70dc818..6020bc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
@@ -79,6 +79,14 @@ public interface CGroupsHandler {
       ResourceHandlerException;
 
   /**
+   * Gets the relative path for the cgroup, independent of a controller, for a
+   * given cgroup id.
+   * @param cGroupId - id of the cgroup
+   * @return path for the cgroup relative to the root of (any) controller.
+   */
+  public String getRelativePathForCGroup(String cGroupId);
+
+  /**
    * Gets the full path for the cgroup, given a controller and a cgroup id
    * @param controller - controller type for the cgroup
    * @param cGroupId - id of the cgroup

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index ff56121..0d71a9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -147,9 +147,9 @@ class CGroupsHandlerImpl implements CGroupsHandler {
           } else {
             String error =
                 new StringBuffer("Mount point Based on mtab file: ")
-                  .append(mtab)
-                  .append(". Controller mount point not writable for: ")
-                  .append(name).toString();
+                    .append(mtab)
+                    .append(". Controller mount point not writable for: ")
+                    .append(name).toString();
 
             LOG.error(error);
             throw new ResourceHandlerException(error);
@@ -272,6 +272,12 @@ class CGroupsHandlerImpl implements CGroupsHandler {
   }
 
   @Override
+  public String getRelativePathForCGroup(String cGroupId) {
+    return new StringBuffer(cGroupPrefix).append("/")
+        .append(cGroupId).toString();
+  }
+
+  @Override
   public String getPathForCGroup(CGroupController controller, String cGroupId) {
     return new StringBuffer(getControllerPath(controller))
         .append('/').append(cGroupPrefix).append("/")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
new file mode 100644
index 0000000..633fa66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -0,0 +1,148 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+
+import java.util.List;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
+  private static final Log LOG = LogFactory
+      .getLog(DefaultLinuxContainerRuntime.class);
+  private Configuration conf;
+  private final PrivilegedOperationExecutor privilegedOperationExecutor;
+
+  public DefaultLinuxContainerRuntime(PrivilegedOperationExecutor
+      privilegedOperationExecutor) {
+    this.privilegedOperationExecutor = privilegedOperationExecutor;
+  }
+
+  @Override
+  public void initialize(Configuration conf)
+      throws ContainerExecutionException {
+    this.conf = conf;
+  }
+
+  @Override
+  public void prepareContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    //nothing to do here at the moment.
+  }
+
+  @Override
+  public void launchContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    PrivilegedOperation launchOp = new PrivilegedOperation(
+        PrivilegedOperation.OperationType.LAUNCH_CONTAINER, (String) null);
+
+    //All of these arguments are expected to be available in the runtime context
+    launchOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
+        ctx.getExecutionAttribute(USER),
+        Integer.toString(PrivilegedOperation.
+            RunAsUserCommand.LAUNCH_CONTAINER.getValue()),
+        ctx.getExecutionAttribute(APPID),
+        ctx.getExecutionAttribute(CONTAINER_ID_STR),
+        ctx.getExecutionAttribute(CONTAINER_WORK_DIR).toString(),
+        ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH).toUri()
+            .getPath(),
+        ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
+        ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+            ctx.getExecutionAttribute(LOCAL_DIRS)),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+            ctx.getExecutionAttribute(LOG_DIRS)),
+        ctx.getExecutionAttribute(RESOURCES_OPTIONS));
+
+    String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
+
+    if (tcCommandFile != null) {
+      launchOp.appendArgs(tcCommandFile);
+    }
+
+    //List<String> -> stored as List -> fetched/converted to List<String>
+    //we can't do better here thanks to type-erasure
+    @SuppressWarnings("unchecked")
+    List<String> prefixCommands = (List<String>) ctx.getExecutionAttribute(
+        CONTAINER_LAUNCH_PREFIX_COMMANDS);
+
+    try {
+      privilegedOperationExecutor.executePrivilegedOperation(prefixCommands,
+            launchOp, null, container.getLaunchContext().getEnvironment(),
+            false);
+    } catch (PrivilegedOperationException e) {
+      LOG.warn("Launch container failed. Exception: ", e);
+
+      throw new ContainerExecutionException("Launch container failed", e
+          .getExitCode(), e.getOutput(), e.getErrorOutput());
+    }
+  }
+
+  @Override
+  public void signalContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    PrivilegedOperation signalOp = new PrivilegedOperation(
+        PrivilegedOperation.OperationType.SIGNAL_CONTAINER, (String) null);
+
+    signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
+        ctx.getExecutionAttribute(USER),
+        Integer.toString(PrivilegedOperation.RunAsUserCommand
+            .SIGNAL_CONTAINER.getValue()),
+        ctx.getExecutionAttribute(PID),
+        Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));
+
+    try {
+      PrivilegedOperationExecutor executor = PrivilegedOperationExecutor
+          .getInstance(conf);
+
+      executor.executePrivilegedOperation(null,
+          signalOp, null, container.getLaunchContext().getEnvironment(),
+          false);
+    } catch (PrivilegedOperationException e) {
+      LOG.warn("Signal container failed. Exception: ", e);
+
+      throw new ContainerExecutionException("Signal container failed", e
+          .getExitCode(), e.getOutput(), e.getErrorOutput());
+    }
+  }
+
+  @Override
+  public void reapContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
new file mode 100644
index 0000000..a59415f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -0,0 +1,110 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+
+import java.util.Map;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
+  private static final Log LOG = LogFactory
+      .getLog(DelegatingLinuxContainerRuntime.class);
+  private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime;
+  private DockerLinuxContainerRuntime dockerLinuxContainerRuntime;
+
+  @Override
+  public void initialize(Configuration conf)
+      throws ContainerExecutionException {
+    PrivilegedOperationExecutor privilegedOperationExecutor =
+        PrivilegedOperationExecutor.getInstance(conf);
+
+    defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime(
+        privilegedOperationExecutor);
+    defaultLinuxContainerRuntime.initialize(conf);
+    dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
+        privilegedOperationExecutor);
+    dockerLinuxContainerRuntime.initialize(conf);
+  }
+
+  private LinuxContainerRuntime pickContainerRuntime(Container container) {
+    Map<String, String> env = container.getLaunchContext().getEnvironment();
+    LinuxContainerRuntime runtime;
+
+    if (DockerLinuxContainerRuntime.isDockerContainerRequested(env)){
+      runtime = dockerLinuxContainerRuntime;
+    } else  {
+      runtime = defaultLinuxContainerRuntime;
+    }
+
+    if (LOG.isInfoEnabled()) {
+      LOG.info("Using container runtime: " + runtime.getClass()
+          .getSimpleName());
+    }
+
+    return runtime;
+  }
+
+  @Override
+  public void prepareContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    LinuxContainerRuntime runtime = pickContainerRuntime(container);
+
+    runtime.prepareContainer(ctx);
+  }
+
+  @Override
+  public void launchContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    LinuxContainerRuntime runtime = pickContainerRuntime(container);
+
+    runtime.launchContainer(ctx);
+  }
+
+  @Override
+  public void signalContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    LinuxContainerRuntime runtime = pickContainerRuntime(container);
+
+    runtime.signalContainer(ctx);
+  }
+
+  @Override
+  public void reapContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    LinuxContainerRuntime runtime = pickContainerRuntime(container);
+
+    runtime.reapContainer(ctx);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
new file mode 100644
index 0000000..2430a78
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -0,0 +1,273 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
+  private static final Log LOG = LogFactory.getLog(
+      DockerLinuxContainerRuntime.class);
+
+  @InterfaceAudience.Private
+  public static final String ENV_DOCKER_CONTAINER_IMAGE =
+      "YARN_CONTAINER_RUNTIME_DOCKER_IMAGE";
+  @InterfaceAudience.Private
+  public static final String ENV_DOCKER_CONTAINER_IMAGE_FILE =
+      "YARN_CONTAINER_RUNTIME_DOCKER_IMAGE_FILE";
+  @InterfaceAudience.Private
+  public static final String ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE =
+      "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE";
+
+
+  private Configuration conf;
+  private DockerClient dockerClient;
+  private PrivilegedOperationExecutor privilegedOperationExecutor;
+
+  public static boolean isDockerContainerRequested(
+      Map<String, String> env) {
+    if (env == null) {
+      return false;
+    }
+
+    String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
+
+    return type != null && type.equals("docker");
+  }
+
+  public DockerLinuxContainerRuntime(PrivilegedOperationExecutor
+      privilegedOperationExecutor) {
+    this.privilegedOperationExecutor = privilegedOperationExecutor;
+  }
+
+  @Override
+  public void initialize(Configuration conf)
+      throws ContainerExecutionException {
+    this.conf = conf;
+    dockerClient = new DockerClient(conf);
+  }
+
+  @Override
+  public void prepareContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+
+  }
+
+  public void addCGroupParentIfRequired(String resourcesOptions,
+      String containerIdStr, DockerRunCommand runCommand)
+      throws ContainerExecutionException {
+    if (resourcesOptions.equals(
+        (PrivilegedOperation.CGROUP_ARG_PREFIX + PrivilegedOperation
+            .CGROUP_ARG_NO_TASKS))) {
+      if (LOG.isInfoEnabled()) {
+        LOG.info("no resource restrictions specified. not using docker's "
+            + "cgroup options");
+      }
+    } else {
+      if (LOG.isInfoEnabled()) {
+        LOG.info("using docker's cgroups options");
+      }
+
+      try {
+        CGroupsHandler cGroupsHandler = ResourceHandlerModule
+            .getCGroupsHandler(conf);
+        String cGroupPath = "/" + cGroupsHandler.getRelativePathForCGroup(
+            containerIdStr);
+
+        if (LOG.isInfoEnabled()) {
+          LOG.info("using cgroup parent: " + cGroupPath);
+        }
+
+        runCommand.setCGroupParent(cGroupPath);
+      } catch (ResourceHandlerException e) {
+        LOG.warn("unable to use cgroups handler. Exception: ", e);
+        throw new ContainerExecutionException(e);
+      }
+    }
+  }
+
+
+  @Override
+  public void launchContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    Map<String, String> environment = container.getLaunchContext()
+        .getEnvironment();
+    String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
+
+    if (imageName == null) {
+      throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE
+          + " not set!");
+    }
+
+    String containerIdStr = container.getContainerId().toString();
+    String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
+    Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
+    //List<String> -> stored as List -> fetched/converted to List<String>
+    //we can't do better here thanks to type-erasure
+    @SuppressWarnings("unchecked")
+    List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
+    @SuppressWarnings("unchecked")
+    List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+    @SuppressWarnings("unchecked")
+    DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,
+        runAsUser, imageName)
+        .detachOnRun()
+        .setContainerWorkDir(containerWorkDir.toString())
+        .setNetworkType("host")
+        .addMountLocation("/etc/passwd", "/etc/password:ro");
+    List<String> allDirs = new ArrayList<>(localDirs);
+
+    allDirs.add(containerWorkDir.toString());
+    allDirs.addAll(logDirs);
+    for (String dir: allDirs) {
+      runCommand.addMountLocation(dir, dir);
+    }
+
+    String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
+
+    /** Disabling docker's cgroup parent support for the time being. Docker
+     * needs to use a more recent libcontainer that supports net_cls. In
+     * addition we also need to revisit current cgroup creation in YARN.
+     */
+    //addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
+
+   Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
+        NM_PRIVATE_CONTAINER_SCRIPT_PATH);
+
+    String disableOverride = environment.get(
+        ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
+
+    if (disableOverride != null && disableOverride.equals("true")) {
+      if (LOG.isInfoEnabled()) {
+        LOG.info("command override disabled");
+      }
+    } else {
+      List<String> overrideCommands = new ArrayList<>();
+      Path launchDst =
+          new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
+
+      overrideCommands.add("bash");
+      overrideCommands.add(launchDst.toUri().getPath());
+      runCommand.setOverrideCommandWithArgs(overrideCommands);
+    }
+
+    String commandFile = dockerClient.writeCommandToTempFile(runCommand,
+        containerIdStr);
+    PrivilegedOperation launchOp = new PrivilegedOperation(
+        PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER, (String)
+        null);
+
+    launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER),
+        Integer.toString(PrivilegedOperation
+            .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()),
+        ctx.getExecutionAttribute(APPID),
+        containerIdStr, containerWorkDir.toString(),
+        nmPrivateContainerScriptPath.toUri().getPath(),
+        ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
+        ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+            localDirs),
+        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+            logDirs),
+        commandFile,
+        resourcesOpts);
+
+    String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
+
+    if (tcCommandFile != null) {
+      launchOp.appendArgs(tcCommandFile);
+    }
+
+    try {
+      privilegedOperationExecutor.executePrivilegedOperation(null,
+          launchOp, null, container.getLaunchContext().getEnvironment(),
+          false);
+    } catch (PrivilegedOperationException e) {
+      LOG.warn("Launch container failed. Exception: ", e);
+
+      throw new ContainerExecutionException("Launch container failed", e
+          .getExitCode(), e.getOutput(), e.getErrorOutput());
+    }
+  }
+
+  @Override
+  public void signalContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    PrivilegedOperation signalOp = new PrivilegedOperation(
+        PrivilegedOperation.OperationType.SIGNAL_CONTAINER, (String) null);
+
+    signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
+        ctx.getExecutionAttribute(USER),
+        Integer.toString(PrivilegedOperation
+            .RunAsUserCommand.SIGNAL_CONTAINER.getValue()),
+        ctx.getExecutionAttribute(PID),
+        Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));
+
+    try {
+      PrivilegedOperationExecutor executor = PrivilegedOperationExecutor
+          .getInstance(conf);
+
+      executor.executePrivilegedOperation(null,
+          signalOp, null, container.getLaunchContext().getEnvironment(),
+          false);
+    } catch (PrivilegedOperationException e) {
+      LOG.warn("Signal container failed. Exception: ", e);
+
+      throw new ContainerExecutionException("Signal container failed", e
+          .getExitCode(), e.getOutput(), e.getErrorOutput());
+    }
+  }
+
+  @Override
+  public void reapContainer(ContainerRuntimeContext ctx)
+      throws ContainerExecutionException {
+
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
new file mode 100644
index 0000000..38aea9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
@@ -0,0 +1,38 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
+
+/** Linux-specific container runtime implementations must implement this
+ * interface.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface LinuxContainerRuntime extends ContainerRuntime {
+  void initialize(Configuration conf) throws ContainerExecutionException;
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
new file mode 100644
index 0000000..d2069a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -0,0 +1,69 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext.Attribute;
+
+import java.util.List;
+import java.util.Map;
+
+public final class LinuxContainerRuntimeConstants {
+  private LinuxContainerRuntimeConstants() {
+  }
+
+  public static final Attribute<Map> LOCALIZED_RESOURCES = Attribute
+      .attribute(Map.class, "localized_resources");
+  public static final Attribute<List> CONTAINER_LAUNCH_PREFIX_COMMANDS =
+      Attribute.attribute(List.class, "container_launch_prefix_commands");
+  public static final Attribute<String> RUN_AS_USER =
+      Attribute.attribute(String.class, "run_as_user");
+  public static final Attribute<String> USER = Attribute.attribute(String.class,
+      "user");
+  public static final Attribute<String> APPID =
+      Attribute.attribute(String.class, "appid");
+  public static final Attribute<String> CONTAINER_ID_STR = Attribute
+      .attribute(String.class, "container_id_str");
+  public static final Attribute<Path> CONTAINER_WORK_DIR = Attribute
+      .attribute(Path.class, "container_work_dir");
+  public static final Attribute<Path> NM_PRIVATE_CONTAINER_SCRIPT_PATH =
+      Attribute.attribute(Path.class, "nm_private_container_script_path");
+  public static final Attribute<Path> NM_PRIVATE_TOKENS_PATH = Attribute
+      .attribute(Path.class, "nm_private_tokens_path");
+  public static final Attribute<Path> PID_FILE_PATH = Attribute.attribute(
+      Path.class, "pid_file_path");
+  public static final Attribute<List> LOCAL_DIRS = Attribute.attribute(
+      List.class, "local_dirs");
+  public static final Attribute<List> LOG_DIRS = Attribute.attribute(
+      List.class, "log_dirs");
+  public static final Attribute<String> RESOURCES_OPTIONS = Attribute.attribute(
+      String.class, "resources_options");
+  public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(
+      String.class, "tc_command_file");
+  public static final Attribute<String> CGROUP_RELATIVE_PATH = Attribute
+      .attribute(String.class, "cgroup_relative_path");
+
+  public static final Attribute<String> PID = Attribute.attribute(
+      String.class, "pid");
+  public static final Attribute<ContainerExecutor.Signal> SIGNAL = Attribute
+      .attribute(ContainerExecutor.Signal.class, "signal");
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
new file mode 100644
index 0000000..faf955f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -0,0 +1,82 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class DockerClient {
+  private static final Log LOG = LogFactory.getLog(DockerClient.class);
+  private static final String TMP_FILE_PREFIX = "docker.";
+  private static final String TMP_FILE_SUFFIX = ".cmd";
+  private final String tmpDirPath;
+
+  public DockerClient(Configuration conf) throws ContainerExecutionException {
+
+    String tmpDirBase = conf.get("hadoop.tmp.dir");
+    if (tmpDirBase == null) {
+      throw new ContainerExecutionException("hadoop.tmp.dir not set!");
+    }
+    tmpDirPath = tmpDirBase + "/nm-docker-cmds";
+
+    File tmpDir = new File(tmpDirPath);
+    if (!(tmpDir.exists() || tmpDir.mkdirs())) {
+      LOG.warn("Unable to create directory: " + tmpDirPath);
+      throw new ContainerExecutionException("Unable to create directory: " +
+          tmpDirPath);
+    }
+  }
+
+  public String writeCommandToTempFile(DockerCommand cmd, String filePrefix)
+      throws ContainerExecutionException {
+    File dockerCommandFile = null;
+    try {
+      dockerCommandFile = File.createTempFile(TMP_FILE_PREFIX + filePrefix,
+          TMP_FILE_SUFFIX, new
+          File(tmpDirPath));
+
+      Writer writer = new OutputStreamWriter(new FileOutputStream(dockerCommandFile),
+          "UTF-8");
+      PrintWriter printWriter = new PrintWriter(writer);
+      printWriter.print(cmd.getCommandWithArguments());
+      printWriter.close();
+
+      return dockerCommandFile.getAbsolutePath();
+    } catch (IOException e) {
+      LOG.warn("Unable to write docker command to temporary file!");
+      throw new ContainerExecutionException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
new file mode 100644
index 0000000..3b76a5c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
@@ -0,0 +1,66 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+
+/** Represents a docker sub-command
+ * e.g 'run', 'load', 'inspect' etc.,
+ */
+
+public abstract class DockerCommand  {
+  private final String command;
+  private final List<String> commandWithArguments;
+
+  protected DockerCommand(String command) {
+    this.command = command;
+    this.commandWithArguments = new ArrayList<>();
+    commandWithArguments.add(command);
+  }
+
+  /** Returns the docker sub-command string being used
+   * e.g 'run'
+   */
+  public final String getCommandOption() {
+    return this.command;
+  }
+
+  /** Add command commandWithArguments - this method is only meant for use by
+   * sub-classes
+   * @param arguments to be added
+   */
+  protected final void addCommandArguments(String... arguments) {
+    this.commandWithArguments.addAll(Arrays.asList(arguments));
+  }
+
+  public String getCommandWithArguments() {
+    return StringUtils.join(" ", commandWithArguments);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c774a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
new file mode 100644
index 0000000..e4d92e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
@@ -0,0 +1,30 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+public class DockerLoadCommand extends DockerCommand {
+  private static final String LOAD_COMMAND = "load";
+
+  public DockerLoadCommand(String localImageFile) {
+    super(LOAD_COMMAND);
+    super.addCommandArguments("--i=" + localImageFile);
+  }
+}


Mime
View raw message