hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject hadoop git commit: HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be order-independent. (Yitong Zhou via gera)
Date Thu, 11 Dec 2014 21:22:20 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk b9f6d0c95 -> 0bcea111e


HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be order-independent.
(Yitong Zhou via gera)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bcea111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bcea111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bcea111

Branch: refs/heads/trunk
Commit: 0bcea111e5daa9a4315346cf6919a4cfc8d90e0d
Parents: b9f6d0c
Author: Gera Shegalov <gera@apache.org>
Authored: Thu Dec 11 12:25:25 2014 -0800
Committer: Gera Shegalov <gera@apache.org>
Committed: Thu Dec 11 13:12:13 2014 -0800

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../hadoop/util/ApplicationClassLoader.java     | 25 ++++++++++++++++----
 .../hadoop/util/TestApplicationClassLoader.java |  8 +++++--
 .../src/main/resources/mapred-default.xml       | 21 ++++++++++++----
 4 files changed, 46 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 47d36e4..d923b87 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -571,6 +571,9 @@ Release 2.7.0 - UNRELEASED
 
     HADOOP-11386. Replace \n by %n in format hadoop-common format strings.
     (Li Lu via wheat9)
+
+    HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be
+    be order-independent. (Yitong Zhou via gera)
     
 Release 2.6.0 - 2014-11-18
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index d2ab015..9f16b61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -216,28 +216,43 @@ public class ApplicationClassLoader extends URLClassLoader {
     return c;
   }
 
+  /**
+   * Checks if a class should be included as a system class.
+   *
+   * A class is a system class if and only if it matches one of the positive
+   * patterns and none of the negative ones.
+   *
+   * @param name the class name to check
+   * @param systemClasses a list of system class configurations.
+   * @return true if the class is a system class
+   */
   public static boolean isSystemClass(String name, List<String> systemClasses) {
+    boolean result = false;
     if (systemClasses != null) {
       String canonicalName = name.replace('/', '.');
       while (canonicalName.startsWith(".")) {
         canonicalName=canonicalName.substring(1);
       }
       for (String c : systemClasses) {
-        boolean result = true;
+        boolean shouldInclude = true;
         if (c.startsWith("-")) {
           c = c.substring(1);
-          result = false;
+          shouldInclude = false;
         }
         if (canonicalName.startsWith(c)) {
           if (   c.endsWith(".")                                   // package
               || canonicalName.length() == c.length()              // class
               ||    canonicalName.length() > c.length()            // nested
                  && canonicalName.charAt(c.length()) == '$' ) {
-            return result;
+            if (shouldInclude) {
+              result = true;
+            } else {
+              return false;
+            }
           }
         }
       }
     }
-    return false;
+    return result;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
index cc16493..be8e61e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
@@ -87,7 +87,7 @@ public class TestApplicationClassLoader {
     assertEquals(jarFile.toURI().toURL(), urls[2]);
     // nofile should be ignored
   }
-  
+
   @Test
   public void testIsSystemClass() {
     testIsSystemClassInternal("");
@@ -112,8 +112,12 @@ public class TestApplicationClassLoader {
         classes("-org.example.Foo,org.example.")));
     assertTrue(isSystemClass("org.example.Bar" + nestedClass,
         classes("-org.example.Foo.,org.example.")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.,-org.example.Foo")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.Foo,-org.example.Foo")));
   }
-  
+
   private List<String> classes(String classes) {
     return Lists.newArrayList(Splitter.on(',').split(classes));
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 00a89c9..6e0deaa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1363,10 +1363,23 @@
    <value></value>
   <description>Used to override the default definition of the system classes for
     the job classloader. The system classes are a comma-separated list of
-    classes that should be loaded from the system classpath, not the
-    user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending
-    in '.' (period) are treated as package names, and names starting with a '-'
-    are treated as negative matches.
+    patterns that indicate whether to load a class from the system classpath,
+    instead from the user-supplied JARs, when mapreduce.job.classloader is
+    enabled.
+
+    A positive pattern is defined as:
+        1. A single class name 'C' that matches 'C' and transitively all nested
+            classes 'C$*' defined in C;
+        2. A package name ending with a '.' (e.g., "com.example.") that matches
+            all classes from that package.
+    A negative pattern is defined by a '-' in front of a positive pattern
+    (e.g., "-com.example.").
+
+    A class is considered a system class if and only if it matches one of the
+    positive patterns and none of the negative ones. More formally:
+    A class is a member of the inclusion set I if it matches one of the positive
+    patterns. A class is a member of the exclusion set E if it matches one of
+    the negative patterns. The set of system classes S = I \ E.
   </description>
 </property>
 


Mime
View raw message