hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1053730 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobConf.java src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
Date Wed, 29 Dec 2010 20:54:19 GMT
Author: todd
Date: Wed Dec 29 20:54:19 2010
New Revision: 1053730

URL: http://svn.apache.org/viewvc?rev=1053730&view=rev
Log:
MAPREDUCE-714. JobConf.findContainingJar unescapes unnecessarily on linux. Contributed by
Todd Lipcon

Added:
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1053730&r1=1053729&r2=1053730&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Dec 29 20:54:19 2010
@@ -453,6 +453,8 @@ Release 0.22.0 - Unreleased
 
     MAPREDUCE-2224. Fix synchronization bugs in JvmManager. (todd)
 
+    MAPREDUCE-714. JobConf.findContainingJar unescapes unnecessarily on linux (todd)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=1053730&r1=1053729&r2=1053730&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed Dec 29 20:54:19
2010
@@ -1833,7 +1833,7 @@ public class JobConf extends Configurati
    * @return a jar file that contains the class, or null.
    * @throws IOException
    */
-  private static String findContainingJar(Class my_class) {
+  static String findContainingJar(Class my_class) {
     ClassLoader loader = my_class.getClassLoader();
     String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
     try {
@@ -1845,6 +1845,13 @@ public class JobConf extends Configurati
           if (toReturn.startsWith("file:")) {
             toReturn = toReturn.substring("file:".length());
           }
+          // URLDecoder is a misnamed class, since it actually decodes
+          // x-www-form-urlencoded MIME type rather than actual
+          // URL encoding (which the file path has). Therefore it would
+          // decode +s to ' 's which is incorrect (spaces are actually
+          // either unencoded or encoded as "%20"). Replace +s first, so
+          // that they are kept sacred during the decoding process.
+          toReturn = toReturn.replaceAll("\\+", "%2B");
           toReturn = URLDecoder.decode(toReturn, "UTF-8");
           return toReturn.replaceAll("!.*$", "");
         }

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java?rev=1053730&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java (added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java Wed Dec
29 20:54:19 2010
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.junit.Test;
+import java.io.File;
+import java.net.URLClassLoader;
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+
+
+import static org.junit.Assert.*;
+
+public class TestJobConf {
+  private static final String JAR_RELATIVE_PATH =
+    "build/test/mapred/testjar/testjob.jar";
+  private static final String CLASSNAME = "ClassWithNoPackage";
+
+  private static String TEST_DIR_WITH_SPECIAL_CHARS =
+    System.getProperty("test.build.data","/tmp") +
+    File.separator + "test jobconf with + and spaces";
+
+  @Test
+  public void testFindContainingJar() throws Exception {
+    testJarAtPath(JAR_RELATIVE_PATH);
+  }
+
+  /**
+   * Test that findContainingJar works correctly even if the
+   * path has a "+" sign or spaces in it
+   */
+  @Test
+  public void testFindContainingJarWithPlus() throws Exception {
+    new File(TEST_DIR_WITH_SPECIAL_CHARS).mkdirs();
+    Configuration conf = new Configuration();
+
+    FileSystem localfs = FileSystem.getLocal(conf);
+
+    FileUtil.copy(localfs, new Path(JAR_RELATIVE_PATH),
+                  localfs, new Path(TEST_DIR_WITH_SPECIAL_CHARS, "test.jar"),
+                  false, true, conf);
+    testJarAtPath(TEST_DIR_WITH_SPECIAL_CHARS + File.separator + "test.jar");
+  }
+
+  /**
+   * Given a path with a jar, make a classloader with that jar on the
+   * classpath, and check that findContainingJar can correctly
+   * identify the path of the jar.
+   */
+  private void testJarAtPath(String path) throws Exception {
+    File jar = new File(path).getAbsoluteFile();
+    assertTrue(jar.exists());
+
+    URL urls[] = new URL[] {
+      jar.toURI().toURL()
+    };
+
+    ClassLoader cl = new URLClassLoader(urls);
+    Class clazz = Class.forName(CLASSNAME, true, cl);
+    assertNotNull(clazz);
+
+    String containingJar = JobConf.findContainingJar(clazz);
+    assertEquals(jar.getAbsolutePath(), containingJar);
+  }
+}
\ No newline at end of file



Mime
View raw message