hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [2/2] hadoop git commit: YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.
Date Wed, 27 May 2015 21:31:37 GMT
YARN-3626. On Windows localized resources are not moved to the front of the classpath when
they should be. Contributed by Craig Welch.

(cherry picked from commit 7bc33b63c78c6ee9a04c0c1511ef16e63f7f3074)
(cherry picked from commit 6ed8989a6f897437ab53c85c42fa8b58fe556359)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02ab202a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02ab202a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02ab202a

Branch: refs/heads/branch-2.7
Commit: 02ab202a2b4cf62a57e494641220fef8c5779698
Parents: fb57a1a
Author: cnauroth <cnauroth@apache.org>
Authored: Wed May 27 14:21:05 2015 -0700
Committer: cnauroth <cnauroth@apache.org>
Committed: Wed May 27 14:21:26 2015 -0700

----------------------------------------------------------------------
 .../mapreduce/v2/app/job/impl/TaskAttemptImpl.java       |  6 ++++++
 .../java/org/apache/hadoop/mapreduce/v2/util/MRApps.java |  5 -----
 .../org/apache/hadoop/yarn/api/ApplicationConstants.java |  8 ++++++++
 .../org/apache/hadoop/yarn/conf/YarnConfiguration.java   | 11 +----------
 .../containermanager/launcher/ContainerLaunch.java       |  9 +++++----
 .../containermanager/launcher/TestContainerLaunch.java   |  8 +++-----
 6 files changed, 23 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 5a8a658..a603a80 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -817,10 +817,16 @@ public abstract class TaskAttemptImpl implements
     // Fill in the fields needed per-container that are missing in the common
     // spec.
 
+    boolean userClassesTakesPrecedence =
+      conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
+
     // Setup environment by cloning from common env.
     Map<String, String> env = commonContainerSpec.getEnvironment();
     Map<String, String> myEnv = new HashMap<String, String>(env.size());
     myEnv.putAll(env);
+    if (userClassesTakesPrecedence) {
+      myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
+    }
     MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
 
     // Set up the launch command

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index f4487af..95bfaec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -241,11 +241,6 @@ public class MRApps extends Apps {
     boolean userClassesTakesPrecedence = 
       conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
 
-    if (userClassesTakesPrecedence) {
-      conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
-        "true");
-    }
-
     String classpathEnvVar =
       conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
         ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index d42cd67..42464da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.api;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -191,6 +192,13 @@ public interface ApplicationConstants {
     HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
 
     /**
+     * $CLASSPATH_PREPEND_DISTCACHE
+     * Private, Windows specific
+     */
+    @Private
+    CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
+
+    /**
      * $CONTAINER_ID
      * Final, exported by NodeManager and non-modifiable by users.
      */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 45fe075..b6b0838 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -611,6 +611,7 @@ public class YarnConfiguration extends Configuration {
       ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
       ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
       ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
+      ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
       ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
   
   /** address of node manager IPC.*/
@@ -1161,16 +1162,6 @@ public class YarnConfiguration extends Configuration {
       + "application.classpath";
 
   /**
-   * Whether or not entries from the distributed cache should be preferred over
-   * the rest of the YARN CLASSPATH
-   */
-  public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
-    YARN_PREFIX + "application.classpath.prepend.distcache";
-
-  public static final boolean
-    DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
-
-  /**
    * Default platform-agnostic CLASSPATH for YARN applications. A
    * comma-separated list of CLASSPATH entries. The parameter expansion marker
    * will be replaced with real parameter expansion marker ('%' for Windows and

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 3100b5e..e27cc2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -735,11 +735,12 @@ public class ContainerLaunch implements Callable<Integer> {
         //jar is created and so they "are lost" and have to be explicitly
         //added to the classpath instead.  This also means that their position
         //is lost relative to other non-distcache classpath entries which will
-        //break things like mapreduce.job.user.classpath.first.
+        //break things like mapreduce.job.user.classpath.first.  An environment
+        //variable can be set to indicate that distcache entries should come
+        //first
 
-        boolean preferLocalizedJars = conf.getBoolean(
-          YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
-          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
+        boolean preferLocalizedJars = Boolean.valueOf(
+          environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
           );
 
         boolean needsSeparator = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index efeeb1d..2856357 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -417,7 +417,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
     userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
     userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
-    userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
+    userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
     containerLaunchContext.setEnvironment(userSetEnv);
     Container container = mock(Container.class);
     when(container.getContainerId()).thenReturn(cId);
@@ -463,14 +463,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     Assert.assertTrue(
       result.get(result.size() - 1).endsWith("userjarlink.jar"));
 
-    //Now move userjar to the front
+    //Then, with user classpath first
+    userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
 
     cId = ContainerId.newContainerId(appAttemptId, 1);
     when(container.getContainerId()).thenReturn(cId);
 
-    conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
-      "true");
-
     launch = new ContainerLaunch(distContext, conf,
         dispatcher, exec, null, container, dirsHandler, containerManager);
 


Mime
View raw message