hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject svn commit: r1471464 - in /hbase/trunk: hbase-client/src/test/java/org/apache/hadoop/hbase/client/ hbase-common/src/main/java/org/apache/hadoop/hbase/util/ hbase-common/src/test/java/org/apache/hadoop/hbase/util/ hbase-server/src/main/java/org/apache/h...
Date Wed, 24 Apr 2013 15:09:49 GMT
Author: jxiang
Date: Wed Apr 24 15:09:48 2013
New Revision: 1471464

URL: http://svn.apache.org/r1471464
Log:
HBASE-8327 Consolidate class loaders

Added:
    hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java   (with props)
    hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java   (with props)
    hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java   (with props)
Removed:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
Modified:
    hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
    hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
    hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java

Modified: hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java?rev=1471464&r1=1471463&r2=1471464&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java (original)
+++ hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java Wed Apr 24 15:09:48 2013
@@ -147,8 +147,8 @@ public class TestGet {
     }
 
     Configuration conf = HBaseConfiguration.create();
-    String localPath = conf.get("hbase.local.dir") + File.separator
-      + "dynamic" + File.separator + "jars" + File.separator;
+    String localPath = conf.get("hbase.local.dir")
+      + File.separator + "jars" + File.separator;
     File jarFile = new File(localPath, "MockFilter.jar");
     jarFile.deleteOnExit();
 

Added: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java?rev=1471464&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java Wed Apr 24 15:09:48 2013
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base class loader that defines couple shared constants used
+ * by sub-classes. It also defined method getClassLoadingLock for parallel
+ * class loading and JDK 1.6 support. This method (getClassLoadingLock)
+ * is similar to the same method in the base class Java ClassLoader
+ * introduced in JDK 1.7, but not in JDK 1.6.
+ */
+@InterfaceAudience.Private
+public class ClassLoaderBase extends URLClassLoader {
+
+  // Maps class name to the corresponding lock object
+  private final ConcurrentHashMap<String, Object> parallelLockMap
+    = new ConcurrentHashMap<String, Object>();
+
+  protected static final String DEFAULT_LOCAL_DIR = "/tmp/hbase-local-dir";
+  protected static final String LOCAL_DIR_KEY = "hbase.local.dir";
+
+  /**
+   * Parent class loader.
+   */
+  protected final ClassLoader parent;
+
+  /**
+   * Creates a DynamicClassLoader that can load classes dynamically
+   * from jar files under a specific folder.
+   *
+   * @param conf the configuration for the cluster.
+   * @param parent the parent ClassLoader to set.
+   */
+  public ClassLoaderBase(final ClassLoader parent) {
+    super(new URL[]{}, parent);
+    Preconditions.checkNotNull(parent, "No parent classloader!");
+    this.parent = parent;
+  }
+
+  /**
+   * Returns the lock object for class loading operations.
+   */
+  protected Object getClassLoadingLock(String className) {
+    Object lock = parallelLockMap.get(className);
+    if (lock != null) {
+      return lock;
+    }
+
+    Object newLock = new Object();
+    lock = parallelLockMap.putIfAbsent(className, newLock);
+    if (lock == null) {
+      lock = newLock;
+    }
+    return lock;
+  }
+}

Propchange: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java?rev=1471464&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java Wed Apr 24 15:09:48 2013
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.concurrent.ConcurrentMap;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * ClassLoader used to load classes for Coprocessor instances.
+ * <p>
+ * This ClassLoader always tries to load classes from the specified coprocessor
+ * jar first actually using URLClassLoader logic before delegating to the parent
+ * ClassLoader, thus avoiding dependency conflicts between HBase's classpath and
+ * classes in the coprocessor jar.
+ * <p>
+ * Certain classes are exempt from being loaded by this ClassLoader because it
+ * would prevent them from being cast to the equivalent classes in the region
+ * server.  For example, the Coprocessor interface needs to be loaded by the
+ * region server's ClassLoader to prevent a ClassCastException when casting the
+ * coprocessor implementation.
+ * <p>
+ * A HDFS path can be used to specify the coprocessor jar. In this case, the jar
+ * will be copied to local at first under some folder under ${hbase.local.dir}/jars/tmp/.
+ * The local copy will be removed automatically when the HBase server instance is
+ * stopped.
+ * <p>
+ * This ClassLoader also handles resource loading.  In most cases this
+ * ClassLoader will attempt to load resources from the coprocessor jar first
+ * before delegating to the parent.  However, like in class loading,
+ * some resources need to be handled differently.  For all of the Hadoop
+ * default configurations (e.g. hbase-default.xml) we will check the parent
+ * ClassLoader first to prevent issues such as failing the HBase default
+ * configuration version check.
+ */
+@InterfaceAudience.Private
+public class CoprocessorClassLoader extends ClassLoaderBase {
+  private static final Log LOG = LogFactory.getLog(CoprocessorClassLoader.class);
+
+  // A temporary place ${hbase.local.dir}/jars/tmp/ to store the local
+  // copy of the jar file and the libraries contained in the jar.
+  private static final String TMP_JARS_DIR = File.separator
+     + "jars" + File.separator + "tmp" + File.separator;
+
+  /**
+   * External class loaders cache keyed by external jar path.
+   * ClassLoader instance is stored as a weak-reference
+   * to allow GC'ing when it is not used
+   * (@see HBASE-7205)
+   */
+  private static final ConcurrentMap<Path, CoprocessorClassLoader> classLoadersCache =
+    new MapMaker().concurrencyLevel(3).weakValues().makeMap();
+
+  /**
+   * If the class being loaded starts with any of these strings, we will skip
+   * trying to load it from the coprocessor jar and instead delegate
+   * directly to the parent ClassLoader.
+   */
+  private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
+    // Java standard library:
+    "com.sun.",
+    "launcher.",
+    "java.",
+    "javax.",
+    "org.ietf",
+    "org.omg",
+    "org.w3c",
+    "org.xml",
+    "sunw.",
+    // logging
+    "org.apache.commons.logging",
+    "org.apache.log4j",
+    "com.hadoop",
+    // Hadoop/HBase/ZK:
+    "org.apache.hadoop",
+    "org.apache.zookeeper",
+  };
+
+  /**
+   * If the resource being loaded matches any of these patterns, we will first
+   * attempt to load the resource with the parent ClassLoader.  Only if the
+   * resource is not found by the parent do we attempt to load it from the coprocessor jar.
+   */
+  private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
+      new Pattern[] {
+    Pattern.compile("^[^-]+-default\\.xml$")
+  };
+
+  /**
+   * Creates a JarClassLoader that loads classes from the given paths.
+   */
+  private CoprocessorClassLoader(ClassLoader parent) {
+    super(parent);
+  }
+
+  private void init(Path path, String pathPrefix,
+      Configuration conf) throws IOException {
+    if (path == null) {
+      throw new IOException("The jar path is null");
+    }
+    if (!path.toString().endsWith(".jar")) {
+      throw new IOException(path.toString() + ": not a jar file?");
+    }
+
+    // Copy the jar to the local filesystem
+    String parentDirPath =
+      conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR;
+    File parentDir = new File(parentDirPath);
+    if (!parentDir.mkdirs() && !parentDir.isDirectory()) {
+      throw new RuntimeException("Failed to create local dir " + parentDir.getPath()
+        + ", CoprocessorClassLoader failed to init");
+    }
+
+    FileSystem fs = path.getFileSystem(conf);
+    File dst = new File(parentDir, "." + pathPrefix + "."
+      + path.getName() + "." + System.currentTimeMillis() + ".jar");
+    fs.copyToLocalFile(path, new Path(dst.toString()));
+    dst.deleteOnExit();
+
+    addURL(dst.getCanonicalFile().toURI().toURL());
+
+    JarFile jarFile = new JarFile(dst.toString());
+    try {
+      Enumeration<JarEntry> entries = jarFile.entries();
+      while (entries.hasMoreElements()) {
+        JarEntry entry = entries.nextElement();
+        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
+          File file = new File(parentDir, "." + pathPrefix + "." + path.getName()
+            + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
+          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
+          file.deleteOnExit();
+          addURL(file.toURI().toURL());
+        }
+      }
+    } finally {
+      jarFile.close();
+    }
+  }
+
+  // This method is used in unit test
+  public static CoprocessorClassLoader getIfCached(final Path path) {
+    if (path == null) return null; // No class loader for null path
+    return classLoadersCache.get(path);
+  }
+
+  // This method is used in unit test
+  public static Collection<? extends ClassLoader> getAllCached() {
+    return classLoadersCache.values();
+  }
+
+  // This method is used in unit test
+  public static void clearCache() {
+    classLoadersCache.clear();
+  }
+
+  /**
+   * Get a CoprocessorClassLoader for a coprocessor jar path from cache.
+   * If not in cache, create one.
+   *
+   * @param path the path to the coprocessor jar file to load classes from
+   * @param parent the parent class loader for exempted classes
+   * @param pathPrefix a prefix used in temp path name to store the jar file locally
+   * @param conf the configuration used to create the class loader, if needed
+   * @return a CoprocessorClassLoader for the coprocessor jar path
+   * @throws IOException
+   */
+  public static CoprocessorClassLoader getClassLoader(final Path path,
+      final ClassLoader parent, final String pathPrefix,
+      final Configuration conf) throws IOException {
+    CoprocessorClassLoader cl = getIfCached(path);
+    if (cl != null) {
+      LOG.debug("Found classloader "+ cl + "for "+ path.toString());
+      return cl;
+    }
+
+    cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
+      @Override
+      public CoprocessorClassLoader run() {
+        return new CoprocessorClassLoader(parent);
+      }
+    });
+
+    cl.init(path, pathPrefix, conf);
+
+    // Cache class loader as a weak value, will be GC'ed when no reference left
+    CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
+    if (prev != null) {
+      // Lost update race, use already added class loader
+      cl = prev;
+    }
+    return cl;
+  }
+
+  @Override
+  public Class<?> loadClass(String name)
+      throws ClassNotFoundException {
+    // Delegate to the parent immediately if this class is exempt
+    if (isClassExempt(name)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Skipping exempt class " + name +
+            " - delegating directly to parent");
+      }
+      return parent.loadClass(name);
+    }
+
+    synchronized (getClassLoadingLock(name)) {
+      // Check whether the class has already been loaded:
+      Class<?> clasz = findLoadedClass(name);
+      if (clasz != null) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Class " + name + " already loaded");
+        }
+      }
+      else {
+        try {
+          // Try to find this class using the URLs passed to this ClassLoader
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Finding class: " + name);
+          }
+          clasz = findClass(name);
+        } catch (ClassNotFoundException e) {
+          // Class not found using this ClassLoader, so delegate to parent
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Class " + name + " not found - delegating to parent");
+          }
+          try {
+            clasz = parent.loadClass(name);
+          } catch (ClassNotFoundException e2) {
+            // Class not found in this ClassLoader or in the parent ClassLoader
+            // Log some debug output before re-throwing ClassNotFoundException
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Class " + name + " not found in parent loader");
+            }
+            throw e2;
+          }
+        }
+      }
+      return clasz;
+    }
+  }
+
+  @Override
+  public URL getResource(String name) {
+    URL resource = null;
+    boolean parentLoaded = false;
+
+    // Delegate to the parent first if necessary
+    if (loadResourceUsingParentFirst(name)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Checking parent first for resource " + name);
+      }
+      resource = super.getResource(name);
+      parentLoaded = true;
+    }
+
+    if (resource == null) {
+      synchronized (getClassLoadingLock(name)) {
+        // Try to find the resource in this jar
+        resource = findResource(name);
+        if ((resource == null) && !parentLoaded) {
+          // Not found in this jar and we haven't attempted to load
+          // the resource in the parent yet; fall back to the parent
+          resource = super.getResource(name);
+        }
+      }
+    }
+    return resource;
+  }
+
+  /**
+   * Determines whether the given class should be exempt from being loaded
+   * by this ClassLoader.
+   * @param name the name of the class to test.
+   * @return true if the class should *not* be loaded by this ClassLoader;
+   * false otherwise.
+   */
+  protected boolean isClassExempt(String name) {
+    for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
+      if (name.startsWith(exemptPrefix)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Determines whether we should attempt to load the given resource using the
+   * parent first before attempting to load the resource using this ClassLoader.
+   * @param name the name of the resource to test.
+   * @return true if we should attempt to load the resource using the parent
+   * first; false if we should attempt to load the resource using this
+   * ClassLoader first.
+   */
+  protected boolean loadResourceUsingParentFirst(String name) {
+    for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
+      if (resourcePattern.matcher(name).matches()) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

Propchange: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java?rev=1471464&r1=1471463&r2=1471464&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java Wed Apr 24 15:09:48 2013
@@ -21,7 +21,6 @@ import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.util.HashMap;
 
 import org.apache.commons.logging.Log;
@@ -34,10 +33,17 @@ import org.apache.hadoop.fs.Path;
 
 /**
  * This is a class loader that can load classes dynamically from new
- * jar files under a configured folder. It always uses its parent class
- * loader to load a class at first. Only if its parent class loader
+ * jar files under a configured folder. The paths to the jar files are
+ * converted to URLs, and URLClassLoader logic is actually used to load
+ * classes. This class loader always uses its parent class loader
+ * to load a class at first. Only if its parent class loader
  * can not load a class, we will try to load it using the logic here.
  * <p>
+ * The configured folder can be a HDFS path. In this case, the jar files
+ * under that folder will be copied to local at first under ${hbase.local.dir}/jars/.
+ * The local copy will be updated if the remote copy is updated, according to its
+ * last modified timestamp.
+ * <p>
  * We can't unload a class already loaded. So we will use the existing
  * jar files we already know to load any class which can't be loaded
  * using the parent class loader. If we still can't load the class from
@@ -50,18 +56,15 @@ import org.apache.hadoop.fs.Path;
  * classes properly.
  */
 @InterfaceAudience.Private
-public class DynamicClassLoader extends URLClassLoader {
+public class DynamicClassLoader extends ClassLoaderBase {
   private static final Log LOG =
       LogFactory.getLog(DynamicClassLoader.class);
 
-  // Dynamic jars are put under ${hbase.local.dir}/dynamic/jars/
+  // Dynamic jars are put under ${hbase.local.dir}/jars/
   private static final String DYNAMIC_JARS_DIR = File.separator
-    + "dynamic" + File.separator + "jars" + File.separator;
+    + "jars" + File.separator;
 
-  /**
-   * Parent class loader used to load any class at first.
-   */
-  private final ClassLoader parent;
+  private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir";
 
   private File localDir;
 
@@ -81,18 +84,18 @@ public class DynamicClassLoader extends 
    */
   public DynamicClassLoader(
       final Configuration conf, final ClassLoader parent) {
-    super(new URL[]{}, parent);
-    this.parent = parent;
+    super(parent);
 
     jarModifiedTime = new HashMap<String, Long>();
-    String localDirPath = conf.get("hbase.local.dir") + DYNAMIC_JARS_DIR;
+    String localDirPath = conf.get(
+      LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR;
     localDir = new File(localDirPath);
     if (!localDir.mkdirs() && !localDir.isDirectory()) {
       throw new RuntimeException("Failed to create local dir " + localDir.getPath()
         + ", DynamicClassLoader failed to init");
     }
 
-    String remotePath = conf.get("hbase.dynamic.jars.dir");
+    String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY);
     if (remotePath == null || remotePath.equals(localDirPath)) {
       remoteDir = null;  // ignore if it is the same as the local path
     } else {
@@ -117,33 +120,35 @@ public class DynamicClassLoader extends 
         LOG.debug("Class " + name + " not found - using dynamical class loader");
       }
 
-      // Check whether the class has already been loaded:
-      Class<?> clasz = findLoadedClass(name);
-      if (clasz != null) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Class " + name + " already loaded");
-        }
-      }
-      else {
-        try {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Finding class: " + name);
-          }
-          clasz = findClass(name);
-        } catch (ClassNotFoundException cnfe) {
-          // Load new jar files if any
+      synchronized (getClassLoadingLock(name)) {
+        // Check whether the class has already been loaded:
+        Class<?> clasz = findLoadedClass(name);
+        if (clasz != null) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Loading new jar files, if any");
+            LOG.debug("Class " + name + " already loaded");
           }
-          loadNewJars();
-
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Finding class again: " + name);
+        }
+        else {
+          try {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Finding class: " + name);
+            }
+            clasz = findClass(name);
+          } catch (ClassNotFoundException cnfe) {
+            // Load new jar files if any
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Loading new jar files, if any");
+            }
+            loadNewJars();
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Finding class again: " + name);
+            }
+            clasz = findClass(name);
           }
-          clasz = findClass(name);
         }
+        return clasz;
       }
-      return clasz;
     }
   }
 

Added: hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java?rev=1471464&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java (added)
+++ hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java Wed Apr 24 15:09:48 2013
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Some utilities to help class loader testing
+ */
+public class ClassLoaderTestHelper {
+  private static final Log LOG = LogFactory.getLog(ClassLoaderTestHelper.class);
+
+  /**
+   * Jar a list of files into a jar archive.
+   *
+   * @param archiveFile the target jar archive
+   * @param tobejared a list of files to be jared
+   */
+  private static boolean createJarArchive(File archiveFile, File[] tobeJared) {
+    try {
+      byte buffer[] = new byte[4096];
+      // Open archive file
+      FileOutputStream stream = new FileOutputStream(archiveFile);
+      JarOutputStream out = new JarOutputStream(stream, new Manifest());
+
+      for (int i = 0; i < tobeJared.length; i++) {
+        if (tobeJared[i] == null || !tobeJared[i].exists()
+            || tobeJared[i].isDirectory()) {
+          continue;
+        }
+
+        // Add archive entry
+        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
+        jarAdd.setTime(tobeJared[i].lastModified());
+        out.putNextEntry(jarAdd);
+
+        // Write file to archive
+        FileInputStream in = new FileInputStream(tobeJared[i]);
+        while (true) {
+          int nRead = in.read(buffer, 0, buffer.length);
+          if (nRead <= 0)
+            break;
+          out.write(buffer, 0, nRead);
+        }
+        in.close();
+      }
+      out.close();
+      stream.close();
+      LOG.info("Adding classes to jar file completed");
+      return true;
+    } catch (Exception ex) {
+      LOG.error("Error: " + ex.getMessage());
+      return false;
+    }
+  }
+
+  /**
+   * Create a test jar for testing purpose for a given class
+   * name with specified code string: save the class to a file,
+   * compile it, and jar it up. If the code string passed in is
+   * null, a bare empty class will be created and used.
+   *
+   * @param testDir the folder under which to store the test class and jar
+   * @param className the test class name
+   * @param code the optional test class code, which can be null.
+   * If null, a bare empty class will be used
+   * @return the test jar file generated
+   */
+  public static File buildJar(String testDir,
+      String className, String code) throws Exception {
+    return buildJar(testDir, className, code, testDir);
+  }
+
+  /**
+   * Create a test jar for testing purpose for a given class
+   * name with specified code string.
+   *
+   * @param testDir the folder under which to store the test class
+   * @param className the test class name
+   * @param code the optional test class code, which can be null.
+   * If null, an empty class will be used
+   * @param folder the folder under which to store the generated jar
+   * @return the test jar file generated
+   */
+  public static File buildJar(String testDir,
+      String className, String code, String folder) throws Exception {
+    String javaCode = code != null ? code : "public class " + className + " {}";
+    Path srcDir = new Path(testDir, "src");
+    File srcDirPath = new File(srcDir.toString());
+    srcDirPath.mkdirs();
+    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
+    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
+    bw.write(javaCode);
+    bw.close();
+
+    // compile it by JavaCompiler
+    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+    ArrayList<String> srcFileNames = new ArrayList<String>();
+    srcFileNames.add(sourceCodeFile.toString());
+    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
+      null);
+    Iterable<? extends JavaFileObject> cu =
+      fm.getJavaFileObjects(sourceCodeFile);
+    List<String> options = new ArrayList<String>();
+    options.add("-classpath");
+    // only add hbase classes to classpath. This is a little bit tricky: assume
+    // the classpath is {hbaseSrc}/target/classes.
+    String currentDir = new File(".").getAbsolutePath();
+    String classpath = currentDir + File.separator + "target"+ File.separator
+      + "classes" + System.getProperty("path.separator")
+      + System.getProperty("java.class.path") + System.getProperty("path.separator")
+      + System.getProperty("surefire.test.class.path");
+
+    options.add(classpath);
+    LOG.debug("Setting classpath to: " + classpath);
+
+    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
+      options, null, cu);
+    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
+
+    // build a jar file by the classes files
+    String jarFileName = className + ".jar";
+    File jarFile = new File(folder, jarFileName);
+    if (!createJarArchive(jarFile,
+        new File[]{new File(srcDir.toString(), className + ".class")})){
+      assertTrue("Build jar file failed.", false);
+    }
+    return jarFile;
+  }
+}

Propchange: hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java?rev=1471464&r1=1471463&r2=1471464&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java (original)
+++ hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java Wed Apr 24 15:09:48 2013
@@ -18,29 +18,14 @@
  */
 package org.apache.hadoop.hbase.util;
 
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-
-import javax.tools.JavaCompiler;
-import javax.tools.JavaFileObject;
-import javax.tools.StandardJavaFileManager;
-import javax.tools.ToolProvider;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.SmallTests;
@@ -62,95 +47,13 @@ public class TestDynamicClassLoader {
     conf.set("hbase.dynamic.jars.dir", TEST_UTIL.getDataTestDir().toString());
   }
 
-  // generate jar file
-  private boolean createJarArchive(File archiveFile, File[] tobeJared) {
-    try {
-      byte buffer[] = new byte[4096];
-      // Open archive file
-      FileOutputStream stream = new FileOutputStream(archiveFile);
-      JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
-      for (int i = 0; i < tobeJared.length; i++) {
-        if (tobeJared[i] == null || !tobeJared[i].exists()
-            || tobeJared[i].isDirectory()) {
-          continue;
-        }
-
-        // Add archive entry
-        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
-        jarAdd.setTime(tobeJared[i].lastModified());
-        out.putNextEntry(jarAdd);
-
-        // Write file to archive
-        FileInputStream in = new FileInputStream(tobeJared[i]);
-        while (true) {
-          int nRead = in.read(buffer, 0, buffer.length);
-          if (nRead <= 0)
-            break;
-          out.write(buffer, 0, nRead);
-        }
-        in.close();
-      }
-      out.close();
-      stream.close();
-      LOG.info("Adding classes to jar file completed");
-      return true;
-    } catch (Exception ex) {
-      LOG.error("Error: " + ex.getMessage());
-      return false;
-    }
-  }
-
-  private File buildJar(
-      String className, String folder) throws Exception {
-    String javaCode = "public class " + className + " {}";
-    Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
-    File srcDirPath = new File(srcDir.toString());
-    srcDirPath.mkdirs();
-    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
-    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
-    bw.write(javaCode);
-    bw.close();
-
-    // compile it by JavaCompiler
-    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
-    ArrayList<String> srcFileNames = new ArrayList<String>();
-    srcFileNames.add(sourceCodeFile.toString());
-    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
-      null);
-    Iterable<? extends JavaFileObject> cu =
-      fm.getJavaFileObjects(sourceCodeFile);
-    List<String> options = new ArrayList<String>();
-    options.add("-classpath");
-    // only add hbase classes to classpath. This is a little bit tricky: assume
-    // the classpath is {hbaseSrc}/target/classes.
-    String currentDir = new File(".").getAbsolutePath();
-    String classpath =
-        currentDir + File.separator + "target"+ File.separator + "classes" +
-        System.getProperty("path.separator") + System.getProperty("java.class.path");
-    options.add(classpath);
-    LOG.debug("Setting classpath to: "+classpath);
-
-    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
-      options, null, cu);
-    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
-
-    // build a jar file by the classes files
-    String jarFileName = className + ".jar";
-    File jarFile = new File(folder, jarFileName);
-    if (!createJarArchive(jarFile,
-        new File[]{new File(srcDir.toString(), className + ".class")})){
-      assertTrue("Build jar file failed.", false);
-    }
-    return jarFile;
-  }
-
   @Test
   public void testLoadClassFromLocalPath() throws Exception {
     ClassLoader parent = TestDynamicClassLoader.class.getClassLoader();
     DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
 
     String className = "TestLoadClassFromLocalPath";
+    deleteClass(className);
     try {
       classLoader.loadClass(className);
       fail("Should not be able to load class " + className);
@@ -159,13 +62,12 @@ public class TestDynamicClassLoader {
     }
 
     try {
-      buildJar(className, localDirPath());
+      String folder = TEST_UTIL.getDataTestDir().toString();
+      ClassLoaderTestHelper.buildJar(folder, className, null, localDirPath());
       classLoader.loadClass(className);
     } catch (ClassNotFoundException cnfe) {
       LOG.error("Should be able to load class " + className, cnfe);
       fail(cnfe.getMessage());
-    } finally {
-      deleteClass(className);
     }
   }
 
@@ -175,6 +77,7 @@ public class TestDynamicClassLoader {
     DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
 
     String className = "TestLoadClassFromAnotherPath";
+    deleteClass(className);
     try {
       classLoader.loadClass(className);
       fail("Should not be able to load class " + className);
@@ -183,30 +86,32 @@ public class TestDynamicClassLoader {
     }
 
     try {
-      buildJar(className, TEST_UTIL.getDataTestDir().toString());
+      String folder = TEST_UTIL.getDataTestDir().toString();
+      ClassLoaderTestHelper.buildJar(folder, className, null);
       classLoader.loadClass(className);
     } catch (ClassNotFoundException cnfe) {
       LOG.error("Should be able to load class " + className, cnfe);
       fail(cnfe.getMessage());
-    } finally {
-      deleteClass(className);
     }
   }
 
   private String localDirPath() {
-    return conf.get("hbase.local.dir") + File.separator
-      + "dynamic" + File.separator + "jars" + File.separator;
+    return conf.get("hbase.local.dir")
+      + File.separator + "jars" + File.separator;
   }
 
   private void deleteClass(String className) throws Exception {
     String jarFileName = className + ".jar";
     File file = new File(TEST_UTIL.getDataTestDir().toString(), jarFileName);
-    file.deleteOnExit();
+    file.delete();
+    assertFalse("Should be deleted: " + file.getPath(), file.exists());
 
     file = new File(conf.get("hbase.dynamic.jars.dir"), jarFileName);
-    file.deleteOnExit();
+    file.delete();
+    assertFalse("Should be deleted: " + file.getPath(), file.exists());
 
     file = new File(localDirPath(), jarFileName);
-    file.deleteOnExit();
+    file.delete();
+    assertFalse("Should be deleted: " + file.getPath(), file.exists());
   }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java?rev=1471464&r1=1471463&r2=1471464&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java Wed Apr 24 15:09:48 2013
@@ -19,39 +19,51 @@
 
 package org.apache.hadoop.hbase.coprocessor;
 
-import com.google.common.collect.MapMaker;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
 import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
 import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.io.IOUtils;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.*;
-import java.util.concurrent.ConcurrentMap;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
 
 /**
  * Provides the common setup framework and runtime services for coprocessor
@@ -73,10 +85,6 @@ public abstract class CoprocessorHost<E 
   public static final String WAL_COPROCESSOR_CONF_KEY =
     "hbase.coprocessor.wal.classes";
 
-  //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
-  private static final String COPROCESSOR_JARS_DIR = File.separator
-      + "coprocessor" + File.separator + "jars" + File.separator;
-
   private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
   /** Ordered set of loaded coprocessors with lock */
   protected SortedSet<E> coprocessors =
@@ -86,15 +94,6 @@ public abstract class CoprocessorHost<E 
   protected String pathPrefix;
   protected volatile int loadSequence;
 
-  /*
-   * External classloaders cache keyed by external jar path.
-   * ClassLoader instance is stored as a weak-reference
-   * to allow GC'ing when no CoprocessorHost is using it
-   * (@see HBASE-7205)
-   */
-  static ConcurrentMap<Path, ClassLoader> classLoadersCache =
-      new MapMaker().concurrencyLevel(3).weakValues().makeMap();
-
   public CoprocessorHost() {
     pathPrefix = UUID.randomUUID().toString();
   }
@@ -175,7 +174,6 @@ public abstract class CoprocessorHost<E 
    * @param conf configuration for coprocessor
    * @throws java.io.IOException Exception
    */
-  @SuppressWarnings("deprecation")
   public E load(Path path, String className, int priority,
       Configuration conf) throws IOException {
     Class<?> implClass = null;
@@ -190,81 +188,8 @@ public abstract class CoprocessorHost<E 
         throw new IOException("No jar path specified for " + className);
       }
     } else {
-      // Have we already loaded the class, perhaps from an earlier region open
-      // for the same table?
-      cl = classLoadersCache.get(path);
-      if (cl != null){
-        LOG.debug("Found classloader "+ cl + "for "+path.toString());
-        try {
-          implClass = cl.loadClass(className);
-        } catch (ClassNotFoundException e) {
-          LOG.info("Class " + className + " needs to be loaded from a file - " +
-              path + ".");
-          // go ahead to load from file system.
-        }
-      }
-    }
-
-    // If not, load
-    if (implClass == null) {
-      if (path == null) {
-        throw new IOException("No jar path specified for " + className);
-      }
-      // copy the jar to the local filesystem
-      if (!path.toString().endsWith(".jar")) {
-        throw new IOException(path.toString() + ": not a jar file?");
-      }
-      FileSystem fs = path.getFileSystem(this.conf);
-      File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
-      parentDir.mkdirs();
-      File dst = new File(parentDir, "." + pathPrefix +
-          "." + className + "." + System.currentTimeMillis() + ".jar");
-      fs.copyToLocalFile(path, new Path(dst.toString()));
-      dst.deleteOnExit();
-
-      // TODO: code weaving goes here
-
-      // TODO: wrap heap allocations and enforce maximum usage limits
-
-      /* TODO: inject code into loop headers that monitors CPU use and
-         aborts runaway user code */
-
-      // load the jar and get the implementation main class
-      // NOTE: Path.toURL is deprecated (toURI instead) but the URLClassLoader
-      // unsurprisingly wants URLs, not URIs; so we will use the deprecated
-      // method which returns URLs for as long as it is available
-      final List<URL> paths = new ArrayList<URL>();
-      URL url = dst.getCanonicalFile().toURL();
-      paths.add(url);
-
-      JarFile jarFile = new JarFile(dst.toString());
-      Enumeration<JarEntry> entries = jarFile.entries();
-      while (entries.hasMoreElements()) {
-        JarEntry entry = entries.nextElement();
-        if (entry.getName().matches("/lib/[^/]+\\.jar")) {
-          File file = new File(parentDir, "." + pathPrefix +
-              "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
-          IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
-          file.deleteOnExit();
-          paths.add(file.toURL());
-        }
-      }
-      jarFile.close();
-
-      cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
-              @Override
-              public CoprocessorClassLoader run() {
-                return new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
-              }
-            });
-
-      // cache cp classloader as a weak value, will be GC'ed when no reference left
-      ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
-      if (prev != null) {
-        //lost update race, use already added classloader
-        cl = prev;
-      }
-
+      cl = CoprocessorClassLoader.getClassLoader(
+        path, getClass().getClassLoader(), pathPrefix, conf);
       try {
         implClass = cl.loadClass(className);
       } catch (ClassNotFoundException e) {

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java?rev=1471464&r1=1471463&r2=1471464&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java Wed Apr 24 15:09:48 2013
@@ -26,13 +26,14 @@ import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
+import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
+import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.RegionLoad;
 
-import javax.tools.*;
 import java.io.*;
 import java.util.*;
 import java.util.jar.*;
@@ -41,6 +42,7 @@ import org.junit.*;
 import org.junit.experimental.categories.Category;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 
@@ -103,90 +105,11 @@ public class TestClassLoading {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  // generate jar file
-  private boolean createJarArchive(File archiveFile, File[] tobeJared) {
-    try {
-      byte buffer[] = new byte[BUFFER_SIZE];
-      // Open archive file
-      FileOutputStream stream = new FileOutputStream(archiveFile);
-      JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
-      for (int i = 0; i < tobeJared.length; i++) {
-        if (tobeJared[i] == null || !tobeJared[i].exists()
-            || tobeJared[i].isDirectory()) {
-          continue;
-        }
-
-        // Add archive entry
-        JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
-        jarAdd.setTime(tobeJared[i].lastModified());
-        out.putNextEntry(jarAdd);
-
-        // Write file to archive
-        FileInputStream in = new FileInputStream(tobeJared[i]);
-        while (true) {
-          int nRead = in.read(buffer, 0, buffer.length);
-          if (nRead <= 0)
-            break;
-          out.write(buffer, 0, nRead);
-        }
-        in.close();
-      }
-      out.close();
-      stream.close();
-      LOG.info("Adding classes to jar file completed");
-      return true;
-    } catch (Exception ex) {
-      LOG.error("Error: " + ex.getMessage());
-      return false;
-    }
-  }
-
-  private File buildCoprocessorJar(String className) throws Exception {
-    // compose a java source file.
-    String javaCode = "import org.apache.hadoop.hbase.coprocessor.*;" +
+  static File buildCoprocessorJar(String className) throws Exception {
+    String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
       "public class " + className + " extends BaseRegionObserver {}";
-    Path baseDir = TEST_UTIL.getDataTestDir();
-    Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
-    File srcDirPath = new File(srcDir.toString());
-    srcDirPath.mkdirs();
-    File sourceCodeFile = new File(srcDir.toString(), className + ".java");
-    BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
-    bw.write(javaCode);
-    bw.close();
-
-    // compile it by JavaCompiler
-    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
-    ArrayList<String> srcFileNames = new ArrayList<String>();
-    srcFileNames.add(sourceCodeFile.toString());
-    StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
-      null);
-    Iterable<? extends JavaFileObject> cu =
-      fm.getJavaFileObjects(sourceCodeFile);
-    List<String> options = new ArrayList<String>();
-    options.add("-classpath");
-    // only add hbase classes to classpath. This is a little bit tricky: assume
-    // the classpath is {hbaseSrc}/target/classes.
-    String currentDir = new File(".").getAbsolutePath();
-    String classpath =
-        currentDir + File.separator + "target"+ File.separator + "classes" +
-        System.getProperty("path.separator") + System.getProperty("java.class.path");
-    options.add(classpath);
-    LOG.debug("Setting classpath to: "+classpath);
-
-    JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
-      options, null, cu);
-    assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
-
-    // build a jar file by the classes files
-    String jarFileName = className + ".jar";
-    File jarFile = new File(baseDir.toString(), jarFileName);
-    if (!createJarArchive(jarFile,
-        new File[]{new File(srcDir.toString(), className + ".class")})){
-      assertTrue("Build jar file failed.", false);
-    }
-
-    return jarFile;
+    return ClassLoaderTestHelper.buildJar(
+      TEST_UTIL.getDataTestDir().toString(), className, code);
   }
 
   @Test
@@ -235,7 +158,7 @@ public class TestClassLoading {
       }
       admin.deleteTable(tableName);
     }
-    CoprocessorHost.classLoadersCache.clear();
+    CoprocessorClassLoader.clearCache();
     byte[] startKey = {10, 63};
     byte[] endKey = {12, 43};
     admin.createTable(htd, startKey, endKey, 4);
@@ -282,22 +205,22 @@ public class TestClassLoading {
     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
     // check if CP classloaders are cached
-    assertTrue(jarFileOnHDFS1 + " was not cached",
-      CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS1));
-    assertTrue(jarFileOnHDFS2 + " was not cached",
-      CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS2));
+    assertNotNull(jarFileOnHDFS1 + " was not cached",
+      CoprocessorClassLoader.getIfCached(pathOnHDFS1));
+    assertNotNull(jarFileOnHDFS2 + " was not cached",
+      CoprocessorClassLoader.getIfCached(pathOnHDFS2));
     //two external jar used, should be one classloader per jar
     assertEquals("The number of cached classloaders should be equal to the number" +
       " of external jar files",
-      2, CoprocessorHost.classLoadersCache.size());
+      2, CoprocessorClassLoader.getAllCached().size());
     //check if region active classloaders are shared across all RS regions
     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
-        CoprocessorHost.classLoadersCache.values());
+      CoprocessorClassLoader.getAllCached());
     for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
-            + " ClassLoader Cache:" + externalClassLoaders
-            + " Region ClassLoaders:" + regionCP.getValue(),
-            externalClassLoaders.containsAll(regionCP.getValue()));
+        + " ClassLoader Cache:" + externalClassLoaders
+        + " Region ClassLoaders:" + regionCP.getValue(),
+        externalClassLoaders.containsAll(regionCP.getValue()));
     }
   }
 



Mime
View raw message