hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sj...@apache.org
Subject hadoop git commit: MAPREDUCE-6577. MR AM unable to load native library without MR_AM_ADMIN_USER_ENV set (sjlee)
Date Wed, 06 Jan 2016 00:07:23 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 51a2e6304 -> b2d02863a


MAPREDUCE-6577. MR AM unable to load native library without MR_AM_ADMIN_USER_ENV set (sjlee)

(cherry picked from commit f6f16118d38fcfe3b724f05fad752cb223f441ec)
(cherry picked from commit 28fd4c70cab27c2872dc6a63ddee223ccf53f21d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2d02863
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2d02863
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2d02863

Branch: refs/heads/branch-2.6
Commit: b2d02863a168910387cd7a3c4b9dee2eb7bb4121
Parents: 51a2e63
Author: Sangjin Lee <sjlee@apache.org>
Authored: Tue Jan 5 15:22:50 2016 -0800
Committer: Sangjin Lee <sjlee@apache.org>
Committed: Tue Jan 5 16:04:28 2016 -0800

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  5 +-
 .../apache/hadoop/mapreduce/MRJobConfig.java    | 21 ++++--
 .../org/apache/hadoop/mapred/YARNRunner.java    |  5 +-
 .../hadoop/mapred/TestMiniMRChildTask.java      | 68 ++++++++++----------
 .../apache/hadoop/mapred/TestYARNRunner.java    | 47 +++++++++++---
 hadoop-project/pom.xml                          |  1 +
 6 files changed, 96 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d02863/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 7fb7b43..b8116bd 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -8,12 +8,15 @@ Release 2.6.4 - UNRELEASED
 
   IMPROVEMENTS
 
-  MAPREDUCE-6436. JobHistory cache issue. (Kai Sasaki via zxu)
+    MAPREDUCE-6436. JobHistory cache issue. (Kai Sasaki via zxu)
 
   OPTIMIZATIONS
 
   BUG FIXES
 
+    MAPREDUCE-6577. MR AM unable to load native library without
+    MR_AM_ADMIN_USER_ENV set (sjlee)
+
 Release 2.6.3 - 2015-12-17
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d02863/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index e09463a..1614bc6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -613,6 +613,16 @@ public interface MRJobConfig {
   public static final String MR_AM_ADMIN_USER_ENV =
       MR_AM_PREFIX + "admin.user.env";
 
+  // although the AM admin user env default should be the same as the task user
+  // env default, there are problems in making it work on Windows currently
+  // MAPREDUCE-6588 should address the issue and set it to a proper non-empty
+  // value
+  public static final String DEFAULT_MR_AM_ADMIN_USER_ENV =
+      Shell.WINDOWS ?
+          "" :
+          "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
+              "/lib/native";
+
   public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
   public static final boolean DEFAULT_MR_AM_PROFILE = false;
   public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
@@ -636,10 +646,13 @@ public interface MRJobConfig {
   public static final String MAPRED_ADMIN_USER_ENV =
       "mapreduce.admin.user.env";
 
-  public final String DEFAULT_MAPRED_ADMIN_USER_ENV = 
-      Shell.WINDOWS ? 
-          "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin":
-          "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native";
+  // the "%...%" macros can be expanded prematurely and are probably not OK
+  // this should be addressed by MAPREDUCE-6588
+  public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
+      Shell.WINDOWS ?
+          "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" :
+          "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
+              "/lib/native";
 
   public static final String WORKDIR = "work";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d02863/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 7b2cf53..5fd178f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -465,13 +465,14 @@ public class YARNRunner implements ClientProtocol {
         conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL,
             MRJobConfig.DEFAULT_SHELL));
 
-    // Add the container working directory at the front of LD_LIBRARY_PATH
+    // Add the container working directory in front of LD_LIBRARY_PATH
     MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
         MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
 
     // Setup the environment variables for Admin first
     MRApps.setEnvFromInputString(environment, 
-        conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
+        conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
+            MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
     // Setup the environment variables (LD_LIBRARY_PATH, etc)
     MRApps.setEnvFromInputString(environment, 
         conf.get(MRJobConfig.MR_AM_ENV), conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d02863/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index 17b5fd2..1631b87 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -424,56 +424,56 @@ public class TestMiniMRChildTask {
   /**
    * To test OS dependent setting of default execution path for a MapRed task.
    * Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
-   * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - for
-   * Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
+   * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH -
+   * for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
    * LD_LIBRARY_PATH
    */
   @Test
   public void testMapRedExecutionEnv() {
-    // test if the env variable can be set
-    try {
-      // Application environment
-      Map<String, String> environment = new HashMap<String, String>();
-      String setupHadoopHomeCommand = Shell.WINDOWS ? 
-          "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" :
-          "HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
-      MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
-
-      // Add the env variables passed by the admin
-      MRApps.setEnvFromInputString(environment, conf.get(
-          MRJobConfig.MAPRED_ADMIN_USER_ENV,
-          MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
-      
-      String executionPaths = environment.get(
-          Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH");
-      String toFind = Shell.WINDOWS ? 
-          "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" : 
-          "/fake/path/to/hadoop/common/home/lib/native";
-      
-      // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
-      assertTrue("execution path does not include the hadoop lib location "
-          + toFind, executionPaths.contains(toFind));
-    } catch (Exception e) {
-      e.printStackTrace();
-      fail("Exception in testing execution environment for MapReduce task");
-      tearDown();
+    // for windows, test if the env variable can be set
+    // this may be removed as part of MAPREDUCE-6588
+    if (Shell.WINDOWS) {
+      try {
+        // Application environment
+        Map<String, String> environment = new HashMap<String, String>();
+        String setupHadoopHomeCommand =
+          "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home";
+        MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
+
+        // Add the env variables passed by the admin
+        MRApps.setEnvFromInputString(environment, conf.get(
+            MRJobConfig.MAPRED_ADMIN_USER_ENV,
+            MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
+
+        String executionPaths = environment.get("PATH");
+        String toFind =
+            "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin";
+
+        // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
+        assertTrue("execution path does not include the hadoop lib location "
+            + toFind, executionPaths.contains(toFind));
+      } catch (Exception e) {
+        e.printStackTrace();
+        fail("Exception in testing execution environment for MapReduce task");
+        tearDown();
+      }
     }
-    
+
     // now launch a mapreduce job to ensure that the child 
     // also gets the configured setting for hadoop lib
     try {
-      
-      JobConf conf = new JobConf(mr.getConfig());      
+
+      JobConf conf = new JobConf(mr.getConfig());
       // initialize input, output directories
       Path inDir = new Path("input");
       Path outDir = new Path("output");
       String input = "The input";
-      
+
       // set config to use the ExecutionEnvCheckMapClass map class
       configure(conf, inDir, outDir, input, 
           ExecutionEnvCheckMapClass.class, IdentityReducer.class);
       launchTest(conf, inDir, outDir, input);
-                 
+
     } catch(Exception e) {
       e.printStackTrace();
       fail("Exception in testing propagation of env setting to child task");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d02863/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 420a95f..d745cff 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -521,16 +522,34 @@ public class TestYARNRunner extends TestCase {
   }
 
   @Test
-  public void testAMStandardEnv() throws Exception {
+  public void testAMStandardEnvWithDefaultLibPath() throws Exception {
+    testAMStandardEnv(false);
+  }
+
+  @Test
+  public void testAMStandardEnvWithCustomLibPath() throws Exception {
+    testAMStandardEnv(true);
+  }
+
+  private void testAMStandardEnv(boolean customLibPath) throws Exception {
+    // the Windows behavior is different and this test currently doesn't really
+    // apply
+    // MAPREDUCE-6588 should revisit this test
+    if (Shell.WINDOWS) {
+      return;
+    }
+
     final String ADMIN_LIB_PATH = "foo";
     final String USER_LIB_PATH = "bar";
     final String USER_SHELL = "shell";
     JobConf jobConf = new JobConf();
+    String pathKey = Environment.LD_LIBRARY_PATH.name();
 
-    jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
-        ADMIN_LIB_PATH);
-    jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
-        + USER_LIB_PATH);
+    if (customLibPath) {
+      jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
+          ADMIN_LIB_PATH);
+      jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
+    }
     jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
 
     YARNRunner yarnRunner = new YARNRunner(jobConf);
@@ -540,15 +559,23 @@ public class TestYARNRunner extends TestCase {
     // make sure PWD is first in the lib path
     ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
     Map<String, String> env = clc.getEnvironment();
-    String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
-    assertNotNull("LD_LIBRARY_PATH not set", libPath);
+    String libPath = env.get(pathKey);
+    assertNotNull(pathKey + " not set", libPath);
     String cps = jobConf.getBoolean(
         MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
         MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
         ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
-    assertEquals("Bad AM LD_LIBRARY_PATH setting",
-        MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
-        + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);
+    String expectedLibPath =
+        MRApps.crossPlatformifyMREnv(conf, Environment.PWD);
+    if (customLibPath) {
+      // append admin libpath and user libpath
+      expectedLibPath += cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH;
+    } else {
+      expectedLibPath += cps +
+          MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV.substring(
+              pathKey.length() + 1);
+    }
+    assertEquals("Bad AM " + pathKey + " setting", expectedLibPath, libPath);
 
     // make sure SHELL is set
     String shell = env.get(Environment.SHELL.name());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d02863/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 89df2ec..b57271f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1111,6 +1111,7 @@
           <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
           <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
           <environmentVariables>
+            <HADOOP_COMMON_HOME>${hadoop.common.build.dir}</HADOOP_COMMON_HOME>
             <!-- HADOOP_HOME required for tests on Windows to find winutils -->
             <HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
             <LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</LD_LIBRARY_PATH>


Mime
View raw message