hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject hadoop git commit: MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. Contributed by Junping Du
Date Wed, 27 Jan 2016 21:23:31 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 8ef73cd4c -> ac43c5635


MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. Contributed by Junping Du

(cherry picked from commit da18bbedaffad1f3ac9c078675583e7cfb68790f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac43c563
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac43c563
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac43c563

Branch: refs/heads/branch-2.6
Commit: ac43c56358a8230484c5ccf81dc496931447f3a4
Parents: 8ef73cd
Author: Jian He <jianhe@apache.org>
Authored: Wed Jan 27 13:16:36 2016 -0800
Committer: Jian He <jianhe@apache.org>
Committed: Wed Jan 27 13:22:59 2016 -0800

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 ++
 .../apache/hadoop/mapreduce/v2/util/MRApps.java |  4 ++
 .../hadoop/mapreduce/v2/util/TestMRApps.java    | 55 ++++++++++++++++++++
 3 files changed, 62 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac43c563/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 514cc44..aeb7597 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -68,6 +68,9 @@ Release 2.6.3 - 2015-12-17
     MAPREDUCE-6549. multibyte delimiters with LineRecordReader cause
     duplicate records (wilfreds via rkanter)
 
+    MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container.
+    (Junping Du via jianhe)
+
 Release 2.6.2 - 2015-10-28
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac43c563/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index af62c1c..4c7a3cf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -258,6 +258,10 @@ public class MRApps extends Apps {
     addClasspathToEnv(environment, classpathEnvVar, conf);
     addClasspathToEnv(environment, hadoopClasspathEnvVar, conf);
 
+    // MAPREDUCE-6619, retain $HADOOP_CLASSPATH
+    MRApps.addToEnvironment(environment, hadoopClasspathEnvVar,
+        System.getenv(hadoopClasspathEnvVar), conf);
+
     if (userClassesTakesPrecedence) {
       MRApps.setMRFrameworkClasspath(environment, conf);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac43c563/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 87473ba..ca30561 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -30,8 +30,10 @@ import static org.mockito.Mockito.when;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.net.URI;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -239,6 +241,12 @@ public class TestMRApps {
       testTGZ.getAbsolutePath())).toString();
     conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
     conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
+    // add hadoop.tgz to env HADOOP_CLASSPATH
+    Map<String, String> newEnv = new HashMap<String, String>();
+    newEnv.put(ApplicationConstants.Environment.HADOOP_CLASSPATH.name(),
+        "hadoop.tgz");
+    setEnv(newEnv);
+
     Map<String, String> environment = new HashMap<String, String>();
     MRApps.setClasspath(environment, conf);
     assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.name()).startsWith(
@@ -268,6 +276,53 @@ public class TestMRApps {
     assertTrue(environment.get(
         ApplicationConstants.Environment.HADOOP_CLASSPATH.name()).
         contains("testTGZ"));
+    assertTrue(environment.get(
+        ApplicationConstants.Environment.HADOOP_CLASSPATH.name()).
+        contains("hadoop.tgz"));
+  }
+
+  // Only set env in runtime but not get persistent in OS (Linux or Windows)
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  private static void setEnv(Map<String, String> newEnv) {
+    try {
+      // Hack for Windows
+      Class<?> processEnvClass =
+          Class.forName("java.lang.ProcessEnvironment");
+      Field theEnvField =
+          processEnvClass.getDeclaredField("theEnvironment");
+      theEnvField.setAccessible(true);
+      Map<String, String> currentEnv =
+          (Map<String, String>)theEnvField.get(null);
+      currentEnv.putAll(newEnv);
+
+      Field caseInsensitiveEnvField =
+          processEnvClass.getDeclaredField("theCaseInsensitiveEnvironment");
+      caseInsensitiveEnvField.setAccessible(true);
+      Map<String, String> ciEnv =
+          (Map<String, String>)caseInsensitiveEnvField.get(null);
+      ciEnv.putAll(newEnv);
+    } catch (NoSuchFieldException e) {
+      // Hack for Linux
+      try {
+        Class[] classes = Collections.class.getDeclaredClasses();
+        Map<String, String> env = System.getenv();
+        for (Class cl : classes) {
+          if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
+            Field field = cl.getDeclaredField("m");
+            field.setAccessible(true);
+            Object obj = field.get(env);
+            Map<String, String> map = (Map<String, String>) obj;
+            map.putAll(newEnv);
+          }
+        }
+      } catch (Exception e1) {
+        LOG.error("Hack env on Linux doesn't work:", e1);
+        throw new RuntimeException(e1);
+      }
+    } catch (Exception e) {
+      LOG.error("Hack env on Windows doesn't work:", e);
+      throw new RuntimeException(e);
+    }
   }
 
  @Test (timeout = 120000)


Mime
View raw message