Author: apurtell
Date: Mon Apr 8 19:49:24 2013
New Revision: 1465745
URL: http://svn.apache.org/r1465745
Log:
HBASE-8158. Amend HBASE-8140 "TableMapReduceUtils#addDependencyJar fails when nested inside
another MR job" (Nick Dimiduk)
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=1465745&r1=1465744&r2=1465745&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
(original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
Mon Apr 8 19:49:24 2013
@@ -579,49 +579,97 @@ public class TableMapReduceUtil {
}
/**
- * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop),
- * finds the Jar for a class or creates it if it doesn't exist. If
- * the class is in a directory in the classpath, it creates a Jar
- * on the fly with the contents of the directory and returns the path
- * to that Jar. If a Jar is created, it is created in
- * the system temporary directory.
- *
- * Otherwise, returns an existing jar that contains a class of the
- * same name.
- *
+ * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds
+ * the Jar for a class or creates it if it doesn't exist. If the class is in
+ * a directory in the classpath, it creates a Jar on the fly with the
+ * contents of the directory and returns the path to that Jar. If a Jar is
+ * created, it is created in the system temporary directory. Otherwise,
+ * returns an existing jar that contains a class of the same name.
* @param my_class the class to find.
- * @return a jar file that contains the class, or null.
+ * @return a jar file that contains the class.
* @throws IOException
*/
private static String findOrCreateJar(Class<?> my_class)
throws IOException {
+ // attempt to locate an existing jar for the class.
+ String jar = findContainingJar(my_class);
+ if (null == jar || jar.isEmpty()) {
+ jar = getJar(my_class);
+ }
+
+ if (null == jar || jar.isEmpty()) {
+ throw new IOException("Cannot locate resource for class " + my_class.getName());
+ }
+
+ LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
+ return jar;
+ }
+
+ /**
+ * Find a jar that contains a class of the same name, if any.
+ * It will return a jar file, even if that is not the first thing
+ * on the class path that has a class with the same name.
+ *
+ * This is shamelessly copied from JobConf
+ *
+ * @param my_class the class to find.
+ * @return a jar file that contains the class, or null.
+ * @throws IOException
+ */
+ private static String findContainingJar(Class<?> my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();)
{
+ URL url = itr.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ return toReturn.replaceAll("!.*$", "");
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Invoke 'getJar' on a JarFinder implementation. Useful for some job configuration
+ * contexts (HBASE-8140) and also for testing on MRv2. First check if we have
+ * HADOOP-9426. Lacking that, fall back to the backport.
+ *
+ * @param my_class the class to find.
+ * @return a jar file that contains the class, or null.
+ */
+ private static String getJar(Class<?> my_class) {
+ String ret = null;
+ String hadoopJarFinder = "org.apache.hadoop.util.JarFinder";
+ Class<?> jarFinder = null;
try {
- Class<?> jarFinder = Class.forName("org.apache.hadoop.util.JarFinder");
- // hadoop-0.23 has a JarFinder class that will create the jar
- // if it doesn't exist. Note that this is needed to run the mapreduce
- // unit tests post-0.23, because mapreduce v2 requires the relevant jars
- // to be in the mr cluster to do output, split, etc. At unit test time,
- // the hbase jars do not exist, so we need to create some.
- Method m = jarFinder.getMethod("getJar", Class.class);
- return (String)m.invoke(null,my_class);
- } catch (InvocationTargetException ite) {
- // function was properly called, but threw it's own exception
- throw new IOException(ite.getCause());
+ LOG.debug("Looking for " + hadoopJarFinder + ".");
+ jarFinder = Class.forName(hadoopJarFinder);
+ LOG.debug(hadoopJarFinder + " found.");
+ Method getJar = jarFinder.getMethod("getJar", Class.class);
+ ret = (String) getJar.invoke(null, my_class);
+ } catch (ClassNotFoundException e) {
+ LOG.debug("Using backported JarFinder.");
+ ret = JarFinder.getJar(my_class);
+ } catch (InvocationTargetException e) {
+ // function was properly called, but threw it's own exception. Unwrap it
+ // and pass it on.
+ throw new RuntimeException(e.getCause());
} catch (Exception e) {
- // ignore all other exceptions. related to reflection failure
+ // toss all other exceptions, related to reflection failure
+ throw new RuntimeException("getJar invocation failed.", e);
}
- LOG.debug("New JarFinder: org.apache.hadoop.util.JarFinder.getJar " +
- "not available. Falling back to backported JarFinder");
- // Use JarFinder because it will construct a jar from class files when
- // one does not exist. This is relevant for cases when an HBase MR job
- // is created in the context of another MR job (particularly common for
- // tools consuming the bulk import APIs). In that case, the dependency
- // jars have already been shipped to and expanded in the job's working
- // directory, so it has no jars to package. We could just construct a
- // classpath from those class files, but we don't know the context: are
- // they on the local filesystem, are they are ephemeral tmp files, &c.
- // Better to package them up and ship them via the normal means.
- return JarFinder.getJar(my_class);
+ return ret;
}
}
|