hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1578140 - in /hadoop/common/branches/branch-2.4/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ hadoop-yarn/hadoop-yarn-ap...
Date Sun, 16 Mar 2014 19:13:56 GMT
Author: vinodkv
Date: Sun Mar 16 19:13:55 2014
New Revision: 1578140

URL: http://svn.apache.org/r1578140
Log:
YARN-1824. Improved NodeManager and clients to be able to handle cross platform application
submissions. Contributed by Jian He.
MAPREDUCE-4052. Improved MapReduce clients to use NodeManagers' ability to handle cross platform
application submissions. Contributed by Jian He.
svn merge --ignore-ancestry -c 1578139 ../branch-2

Modified:
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
    hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/CHANGES.txt Sun Mar 16 19:13:55
2014
@@ -250,6 +250,9 @@ Release 2.4.0 - UNRELEASED
     YARN-1658. Modified web-app framework to let standby RMs redirect
     web-service calls to the active RM. (Cindy Li via vinodkv)
 
+    YARN-1824. Improved NodeManager and clients to be able to handle cross
+    platform application submissions. (Jian He via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
Sun Mar 16 19:13:55 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
 
@@ -59,6 +60,39 @@ public interface ApplicationConstants {
    */
   public static final String LOG_DIR_EXPANSION_VAR = "<LOG_DIR>";
 
+  /**
+   * This constant is used to construct class path and it will be replaced with
+   * real class path separator(':' for Linux and ';' for Windows) by
+   * NodeManager on container launch. User has to use this constant to construct
+   * class path if user wants cross-platform practice i.e. submit an application
+   * from a Windows client to a Linux/Unix server or vice versa.
+   */
+  @Public
+  @Unstable
+  public static final String CLASS_PATH_SEPARATOR= "<CPS>";
+
+  /**
+   * The following two constants are used to expand parameter and it will be
+   * replaced with real parameter expansion marker ('%' for Windows and '$' for
+   * Linux) by NodeManager on container launch. For example: {{VAR}} will be
+   * replaced as $VAR on Linux, and %VAR% on Windows. User has to use this
+   * constant to construct class path if user wants cross-platform practice i.e.
+   * submit an application from a Windows client to a Linux/Unix server or vice
+   * versa.
+   */
+  @Public
+  @Unstable
+  public static final String PARAMETER_EXPANSION_LEFT="{{";
+
+  /**
+   * User has to use this constant to construct class path if user wants
+   * cross-platform practice i.e. submit an application from a Windows client to
+   * a Linux/Unix server or vice versa.
+   */
+  @Public
+  @Unstable
+  public static final String PARAMETER_EXPANSION_RIGHT="}}";
+
   public static final String STDERR = "stderr";
 
   public static final String STDOUT = "stdout";
@@ -206,7 +240,15 @@ public interface ApplicationConstants {
     public String toString() {
       return variable;
     }
-    
+
+    /**
+     * Expand the environment variable based on client OS environment variable
+     * expansion syntax (e.g. $VAR for Linux and %VAR% for Windows).
+     * <p>
+     * Note: Use $$() method for cross-platform practice i.e. submit an
+     * application from a Windows client to a Linux/Unix server or vice versa.
+     * </p>
+     */
     public String $() {
       if (Shell.WINDOWS) {
         return "%" + variable + "%";
@@ -214,5 +256,18 @@ public interface ApplicationConstants {
         return "$" + variable;
       }
     }
+
+    /**
+     * Expand the environment variable in platform-agnostic syntax. The
+     * parameter expansion marker "{{VAR}}" will be replaced with real parameter
+     * expansion marker ('%' for Windows and '$' for Linux) by NodeManager on
+     * container launch. For example: {{VAR}} will be replaced as $VAR on Linux,
+     * and %VAR% on Windows.
+     */
+    @Public
+    @Unstable
+    public String $$() {
+      return PARAMETER_EXPANSION_LEFT + variable + PARAMETER_EXPANSION_RIGHT;
+    }
   }
 }

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Sun Mar 16 19:13:55 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.HadoopIllegalAr
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.net.NetUtils;
@@ -955,8 +956,39 @@ public class YarnConfiguration extends C
       + "application.classpath";
 
   /**
-   * Default CLASSPATH for YARN applications. A comma-separated list of
-   * CLASSPATH entries
+   * Default platform-agnostic CLASSPATH for YARN applications. A
+   * comma-separated list of CLASSPATH entries. The parameter expansion marker
+   * will be replaced with real parameter expansion marker ('%' for Windows and
+   * '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
+   * will be replaced as $VAR on Linux, and %VAR% on Windows.
+   */
+  @Public
+  @Unstable
+  public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= {
+      ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(),
+      ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
+          + "/share/hadoop/common/*",
+      ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
+          + "/share/hadoop/common/lib/*",
+      ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
+          + "/share/hadoop/hdfs/*",
+      ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
+          + "/share/hadoop/hdfs/lib/*",
+      ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
+          + "/share/hadoop/yarn/*",
+      ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
+          + "/share/hadoop/yarn/lib/*" };
+  /**
+   * <p>
+   * Default platform-specific CLASSPATH for YARN applications. A
+   * comma-separated list of CLASSPATH entries constructed based on the client
+   * OS environment expansion syntax.
+   * </p>
+   * <p>
+   * Note: Use {@link DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+   * cross-platform practice i.e. submit an application from a Windows client to
+   * a Linux/Unix server or vice versa.
+   * </p>
    */
   public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
       ApplicationConstants.Environment.HADOOP_CONF_DIR.$(),

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
Sun Mar 16 19:13:55 2014
@@ -47,6 +47,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
@@ -74,7 +76,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -85,6 +86,7 @@ import org.apache.hadoop.yarn.client.api
 import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
@@ -223,8 +225,9 @@ public class ApplicationMaster {
   private long shellScriptPathLen = 0;
 
   // Hardcoded path to shell script in launch container's local env
-  private static final String ExecShellStringPath = "ExecShellScript.sh";
-  private static final String ExecBatScripStringtPath = "ExecBatScript.bat";
+  private static final String ExecShellStringPath = Client.SCRIPT_PATH + ".sh";
+  private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH
+      + ".bat";
 
   // Hardcoded path to custom log_properties
   private static final String log4jPath = "log4j.properties";
@@ -846,15 +849,29 @@ public class ApplicationMaster {
       // In this scenario, if a shell script is specified, we need to have it
       // copied and made available to the container.
       if (!shellScriptPath.isEmpty()) {
+        Path renamedSchellScriptPath = null;
+        if (Shell.WINDOWS) {
+          renamedSchellScriptPath = new Path(shellScriptPath + ".bat");
+        } else {
+          renamedSchellScriptPath = new Path(shellScriptPath + ".sh");
+        }
+        try {
+          FileSystem fs = renamedSchellScriptPath.getFileSystem(conf);
+          fs.rename(new Path(shellScriptPath), renamedSchellScriptPath);
+        } catch (IOException e) {
+          LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename");
+          throw new YarnRuntimeException(e);
+        }
+
         LocalResource shellRsrc = Records.newRecord(LocalResource.class);
         shellRsrc.setType(LocalResourceType.FILE);
         shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
         try {
           shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(
-              shellScriptPath)));
+            renamedSchellScriptPath.toString())));
         } catch (URISyntaxException e) {
           LOG.error("Error when trying to use shell script path specified"
-              + " in env, path=" + shellScriptPath);
+              + " in env, path=" + renamedSchellScriptPath);
           e.printStackTrace();
 
           // A failure scenario on bad input such as invalid shell script path

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Sun Mar 16 19:13:55 2014
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.applications.distributedshell;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -49,7 +48,6 @@ import org.apache.hadoop.io.DataOutputBu
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -177,8 +175,7 @@ public class Client {
   // Hardcoded path to custom log_properties
   private static final String log4jPath = "log4j.properties";
 
-  private static final String linuxShellPath = "ExecShellScript.sh";
-  private static final String windowBatPath = "ExecBatScript.bat";
+  public static final String SCRIPT_PATH = "ExecScript";
 
   /**
    * @param args Command line arguments 
@@ -492,8 +489,7 @@ public class Client {
     if (!shellScriptPath.isEmpty()) {
       Path shellSrc = new Path(shellScriptPath);
       String shellPathSuffix =
-          appName + "/" + appId.getId() + "/"
-              + (Shell.WINDOWS ? windowBatPath : linuxShellPath);
+          appName + "/" + appId.getId() + "/" + SCRIPT_PATH;
       Path shellDst =
           new Path(fs.getHomeDirectory(), shellPathSuffix);
       fs.copyFromLocalFile(false, true, shellSrc, shellDst);
@@ -535,15 +531,16 @@ public class Client {
     // It should be provided out of the box. 
     // For now setting all required classpaths including
     // the classpath to "." for the application jar
-    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$())
-      .append(File.pathSeparatorChar).append("./*");
+    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$())
+      .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
     for (String c : conf.getStrings(
         YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-      classPathEnv.append(File.pathSeparatorChar);
+        YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
+      classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
       classPathEnv.append(c.trim());
     }
-    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
+    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append(
+      "./log4j.properties");
 
     // add the runtime classpath needed for tests to work
     if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
@@ -560,7 +557,7 @@ public class Client {
 
     // Set java executable command 
     LOG.info("Setting up app master command");
-    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+    vargs.add(Environment.JAVA_HOME.$$() + "/bin/java");
     // Set Xmx based on am memory size
     vargs.add("-Xmx" + amMemory + "m");
     // Set class name 

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Sun Mar 16 19:13:55 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringInterner;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
@@ -70,7 +71,7 @@ public class Apps {
   }
 
   public static void setEnvFromInputString(Map<String, String> env,
-      String envString) {
+      String envString,  String classPathSeparator) {
     if (envString != null && envString.length() > 0) {
       String childEnvs[] = envString.split(",");
       Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
@@ -92,7 +93,7 @@ public class Apps {
           m.appendReplacement(sb, Matcher.quoteReplacement(replace));
         }
         m.appendTail(sb);
-        addToEnvironment(env, parts[0], sb.toString());
+        addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
       }
     }
   }
@@ -101,14 +102,19 @@ public class Apps {
   @Unstable
   public static void addToEnvironment(
       Map<String, String> environment,
-      String variable, String value) {
+      String variable, String value, String classPathSeparator) {
     String val = environment.get(variable);
     if (val == null) {
       val = value;
     } else {
-      val = val + File.pathSeparator + value;
+      val = val + classPathSeparator + value;
     }
     environment.put(StringInterner.weakIntern(variable), 
         StringInterner.weakIntern(val));
   }
+
+  public static String crossPlatformify(String var) {
+    return ApplicationConstants.PARAMETER_EXPANSION_LEFT + var
+        + ApplicationConstants.PARAMETER_EXPANSION_RIGHT;
+  }
 }

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Sun Mar 16 19:13:55 2014
@@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
+import com.google.common.annotations.VisibleForTesting;
+
 public class ContainerLaunch implements Callable<Integer> {
 
   private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
@@ -124,6 +126,25 @@ public class ContainerLaunch implements 
             YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
   }
 
+  @VisibleForTesting
+  public static String expandEnvironment(String var,
+      Path containerLogDir) {
+    var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+      containerLogDir.toString());
+    var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
+      File.pathSeparator);
+
+    // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
+    // as %VAR% and on Linux replaced as "$VAR"
+    if (Shell.WINDOWS) {
+      var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
+    } else {
+      var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
+      var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
+    }
+    return var;
+  }
+
   @Override
   @SuppressWarnings("unchecked") // dispatcher not typed
   public Integer call() {
@@ -165,8 +186,7 @@ public class ContainerLaunch implements 
           dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
       for (String str : command) {
         // TODO: Should we instead work via symlinks without this grammar?
-        newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
-            containerLogDir.toString()));
+        newCmds.add(expandEnvironment(str, containerLogDir));
       }
       launchContext.setCommands(newCmds);
 
@@ -174,11 +194,8 @@ public class ContainerLaunch implements 
       // Make a copy of env to iterate & do variable expansion
       for (Entry<String, String> entry : environment.entrySet()) {
         String value = entry.getValue();
-        entry.setValue(
-            value.replace(
-                ApplicationConstants.LOG_DIR_EXPANSION_VAR,
-                containerLogDir.toString())
-            );
+        value = expandEnvironment(value, containerLogDir);
+        entry.setValue(value);
       }
       // /////////////////////////// End of variable expansion
 
@@ -647,12 +664,9 @@ public class ContainerLaunch implements 
     }
 
     // variables here will be forced in, even if the container has specified them.
-    Apps.setEnvFromInputString(
-      environment,
-      conf.get(
-        YarnConfiguration.NM_ADMIN_USER_ENV,
-        YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV)
-    );
+    Apps.setEnvFromInputString(environment, conf.get(
+      YarnConfiguration.NM_ADMIN_USER_ENV,
+      YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV), File.pathSeparator);
 
     // TODO: Remove Windows check and use this approach on all platforms after
     // additional testing.  See YARN-358.

Modified: hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1578140&r1=1578139&r2=1578140&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
Sun Mar 16 19:13:55 2014
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.no
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.spy;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -48,6 +47,7 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -73,12 +73,12 @@ import org.apache.hadoop.yarn.security.C
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
 import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
@@ -287,6 +287,31 @@ public class TestContainerLaunch extends
     }
   }
 
+  @Test(timeout = 10000)
+  public void testEnvExpansion() throws IOException {
+    Path logPath = new Path("/nm/container/logs");
+    String input =
+        Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
+            + ApplicationConstants.CLASS_PATH_SEPARATOR
+            + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
+            + ApplicationConstants.CLASS_PATH_SEPARATOR
+            + Apps.crossPlatformify("HADOOP_LOG_HOME")
+            + ApplicationConstants.LOG_DIR_EXPANSION_VAR;
+
+    String res = ContainerLaunch.expandEnvironment(input, logPath);
+
+    if (Shell.WINDOWS) {
+      Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
+          + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
+          + "%HADOOP_LOG_HOME%/nm/container/logs", res);
+    } else {
+      Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
+          + "$HADOOP_HOME/share/hadoop/common/lib/*:"
+          + "$HADOOP_LOG_HOME/nm/container/logs", res);
+    }
+    System.out.println(res);
+  }
+
   @Test (timeout = 20000)
   public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException {
 



Mime
View raw message