hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/2] hbase git commit: HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)
Date Mon, 01 Aug 2016 23:04:51 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 27c583467 -> 6dfdc0d89


HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

Amending-Author: Andrew Purtell <apurtell@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f750acb7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f750acb7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f750acb7

Branch: refs/heads/0.98
Commit: f750acb751c5cc3b4a3292451c0504942b62bf39
Parents: 27c5834
Author: Jerry He <jerryjch@apache.org>
Authored: Sat Jul 9 18:18:22 2016 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Mon Aug 1 15:52:44 2016 -0700

----------------------------------------------------------------------
 .../hbase/util/CoprocessorClassLoader.java      | 74 +++++++++++++-------
 .../hbase/util/TestCoprocessorClassLoader.java  | 43 ++++++++++++
 2 files changed, 93 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/f750acb7/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index fb2a127..9b65807 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -39,6 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
 import com.google.common.base.Preconditions;
@@ -155,7 +159,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
     super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
       Configuration conf) throws IOException {
     // Copy the jar to the local filesystem
     String parentDirStr =
@@ -173,31 +177,53 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       }
     }
 
-    FileSystem fs = path.getFileSystem(conf);
-    File dst = new File(parentDirStr, "." + pathPrefix + "."
-      + path.getName() + "." + System.currentTimeMillis() + ".jar");
-    fs.copyToLocalFile(path, new Path(dst.toString()));
-    dst.deleteOnExit();
-
-    addURL(dst.getCanonicalFile().toURI().toURL());
+    FileSystem fs = pathPattern.getFileSystem(conf);
+    Path pathPattern1 = fs.isDirectory(pathPattern) ?
+      new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a directory is
specified
+    FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all files that match
the pattern
+    if (fileStatuses == null || fileStatuses.length == 0) {  // if no one matches
+      throw new FileNotFoundException(pathPattern1.toString());
+    } else {
+      boolean validFileEncountered = false;
+      for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file that match
the pattern
+        if (fs.isFile(path)) {  // only process files, skip for directories
+          File dst = new File(parentDirStr, "." + pathPrefix + "."
+            + path.getName() + "." + System.currentTimeMillis() + ".jar");
+          fs.copyToLocalFile(path, new Path(dst.toString()));
+          dst.deleteOnExit();
+
+          addURL(dst.getCanonicalFile().toURI().toURL());
+
+          JarFile jarFile = new JarFile(dst.toString());
+          try {
+            Enumeration<JarEntry> entries = jarFile.entries();  // get entries inside
a jar file
+            while (entries.hasMoreElements()) {
+              JarEntry entry = entries.nextElement();
+              Matcher m = libJarPattern.matcher(entry.getName());
+              if (m.matches()) {
+                File file = new File(parentDirStr, "." + pathPrefix + "."
+                  + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+                FileOutputStream outStream = new FileOutputStream(file);
+                try {
+                  IOUtils.copyBytes(jarFile.getInputStream(entry),
+                    outStream, conf, true);
+                } finally {
+                  outStream.close();
+                }
+                file.deleteOnExit();
+                addURL(file.toURI().toURL());
+              }
+            }
+          } finally {
+            jarFile.close();
+          }
 
-    JarFile jarFile = new JarFile(dst.toString());
-    try {
-      Enumeration<JarEntry> entries = jarFile.entries();
-      while (entries.hasMoreElements()) {
-        JarEntry entry = entries.nextElement();
-        Matcher m = libJarPattern.matcher(entry.getName());
-        if (m.matches()) {
-          File file = new File(parentDirStr, "." + pathPrefix + "."
-            + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
-          IOUtils.copyBytes(jarFile.getInputStream(entry),
-            new FileOutputStream(file), conf, true);
-          file.deleteOnExit();
-          addURL(file.toURI().toURL());
+          validFileEncountered = true;  // Set to true when encountering a file
         }
       }
-    } finally {
-      jarFile.close();
+      if (validFileEncountered == false) {  // all items returned by globStatus() are directories
+        throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+      }
     }
   }
 
@@ -238,7 +264,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       return cl;
     }
 
-    if (!pathStr.endsWith(".jar")) {
+    if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
       throw new IOException(pathStr + ": not a jar file?");
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/f750acb7/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
index f4b2002..e1048da 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -108,4 +109,46 @@ public class TestCoprocessorClassLoader {
     }
     fail("Could not find the expected lib jar file");
   }
+
+  // HBASE-14548
+  @Test
+  public void testDirectoryAndWildcard() throws Exception {
+    String testClassName = "TestClass";
+    String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+    System.out.println(dataTestDir);
+    String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+    ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+    ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+    CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+    CoprocessorClassLoader coprocessorClassLoader = null;
+    Path testPath = null;
+
+    // Directory
+    testPath = new Path(localDirContainingJar);
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1",
conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.jar
+    testPath = new Path(localDirContainingJar, "*.jar");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2",
conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.j*
+    testPath = new Path(localDirContainingJar, "*.j*");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3",
conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+  }
+
+  /**
+   * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+   * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+   * @param className the expected class to be loaded by the coprocessorClassLoader
+   * @throws ClassNotFoundException
+   */
+  private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader,
String className)
+    throws ClassNotFoundException{
+    assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+    assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+  }
 }


Mime
View raw message