hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r695844 - in /hadoop/core/trunk: ./ src/core/org/apache/hadoop/util/ src/test/org/apache/hadoop/mapred/ src/test/testshell/
Date Tue, 16 Sep 2008 12:47:39 GMT
Author: ddas
Date: Tue Sep 16 05:47:39 2008
New Revision: 695844

URL: http://svn.apache.org/viewvc?rev=695844&view=rev
Log:
HADOOP-3570. Includes user specified libjar files in the client side classpath path. Contributed
by Sharad Agarwal.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
    hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
    hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Sep 16 05:47:39 2008
@@ -559,6 +559,9 @@
     HADOOP-4099. Fix null pointer when using HFTP from an 0.18 server.
     (dhruba via omalley)
 
+    HADOOP-3570. Includes user specified libjar files in the client side 
+    classpath path. (Sharad Agarwal via ddas)
+
 Release 0.18.1 - 2008-09-17
 
   IMPROVEMENTS

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java Tue Sep 16
05:47:39 2008
@@ -21,6 +21,8 @@
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -228,6 +230,11 @@
       if (line.hasOption("libjars")) {
         conf.set("tmpjars", 
                  validateFiles(line.getOptionValue("libjars"), conf));
+        //setting libjars in client classpath
+        ClassLoader loader = new URLClassLoader(getLibJars(conf), 
+            conf.getClassLoader());
+        Thread.currentThread().setContextClassLoader(loader);
+        conf.setClassLoader(loader);
       }
       if (line.hasOption("files")) {
         conf.set("tmpfiles", 
@@ -249,6 +256,26 @@
     }
     conf.setBoolean("mapred.used.genericoptionsparser", true);
   }
+  
+  /**
+   * If libjars are set in the conf, parse the libjars.
+   * @param conf
+   * @return libjar urls
+   * @throws IOException
+   */
+  public static URL[] getLibJars(Configuration conf) throws IOException {
+    String jars = conf.get("tmpjars");
+    if(jars==null) {
+      return null;
+    }
+    String[] files = jars.split(",");
+    URL[] cp = new URL[files.length];
+    for (int i=0;i<cp.length;i++) {
+      Path tmp = new Path(files[i]);
+      cp[i] = FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL();
+    }
+    return cp;
+  }
 
   /**
    * takes input as a comma separated list of files

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java Tue Sep 16 05:47:39 2008
@@ -27,6 +27,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.mapred.JobClient;
 
 /** Run a Hadoop job jar. */
 public class RunJar {
@@ -141,6 +142,15 @@
         classPath.add(libs[i].toURL());
       }
     }
+    
+    //adding libjars to the classpath
+    Configuration conf = JobClient.getCommandLineConfig();
+    URL[] libJars = GenericOptionsParser.getLibJars(conf);
+    if(libJars!=null) {
+      for(URL url : libJars){
+        classPath.add(url);
+      }
+    }
     ClassLoader loader =
       new URLClassLoader(classPath.toArray(new URL[0]));
 

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
Tue Sep 16 05:47:39 2008
@@ -68,8 +68,15 @@
       args[3] = "build/test/testjar/testjob.jar";
       args[4] = input.toString();
       args[5] = output.toString();
-      int ret = ToolRunner.run(mr.createJobConf(),
+      
+      JobConf jobConf = mr.createJobConf();
+      //before running the job, verify that libjar is not in client classpath
+      assertTrue("libjar not in client classpath", loadLibJar(jobConf)==null);
+      int ret = ToolRunner.run(jobConf,
                                new testshell.ExternalMapReduce(), args);
+      //after running the job, verify that libjar is in the client classpath
+      assertTrue("libjar added to client classpath", loadLibJar(jobConf)!=null);
+      
       assertTrue("not failed ", ret != -1);
       f.delete();
       thisbuildDir.delete();
@@ -78,4 +85,13 @@
       if (mr != null) {mr.shutdown();};
     }
   }
+  
+  @SuppressWarnings("unchecked")
+  private Class loadLibJar(JobConf jobConf) {
+    try {
+      return jobConf.getClassByName("testjar.ClassWordCount");
+    } catch (ClassNotFoundException e) {
+      return null;
+    }
+  }
 }

Modified: hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java (original)
+++ hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java Tue Sep 16 05:47:39 2008
@@ -104,6 +104,16 @@
     Path outDir = new Path(argv[1]);
     Path input = new Path(argv[0]);
     JobConf testConf = new JobConf(getConf(), ExternalMapReduce.class);
+    
+    //try to load a class from libjar
+    try {
+      testConf.getClassByName("testjar.ClassWordCount");
+    } catch (ClassNotFoundException e) {
+      System.out.println("Could not find class from libjar");
+      return -1;
+    }
+    
+    
     testConf.setJobName("external job");
     FileInputFormat.setInputPaths(testConf, input);
     FileOutputFormat.setOutputPath(testConf, outDir);



Mime
View raw message