hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject [05/50] [abbrv] hbase git commit: HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)
Date Mon, 11 Jul 2016 23:47:58 GMT
HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63296978
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63296978
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63296978

Branch: refs/heads/HBASE-14850
Commit: 632969787aa9e9e1d73b83449b93e21f91110b73
Parents: 496fd98
Author: Jerry He <jerryjch@apache.org>
Authored: Sat Jul 9 18:00:41 2016 -0700
Committer: Jerry He <jerryjch@apache.org>
Committed: Sat Jul 9 18:01:49 2016 -0700

----------------------------------------------------------------------
 .../hbase/util/CoprocessorClassLoader.java      | 72 +++++++++++++-------
 .../hbase/util/TestCoprocessorClassLoader.java  | 43 ++++++++++++
 src/main/asciidoc/_chapters/cp.adoc             |  9 ++-
 3 files changed, 97 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 11016c3..c3635cb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
     super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
       Configuration conf) throws IOException {
     // Copy the jar to the local filesystem
     String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       }
     }
 
-    FileSystem fs = path.getFileSystem(conf);
-    File dst = new File(parentDirStr, "." + pathPrefix + "."
-      + path.getName() + "." + System.currentTimeMillis() + ".jar");
-    fs.copyToLocalFile(path, new Path(dst.toString()));
-    dst.deleteOnExit();
-
-    addURL(dst.getCanonicalFile().toURI().toURL());
-
-    JarFile jarFile = new JarFile(dst.toString());
-    try {
-      Enumeration<JarEntry> entries = jarFile.entries();
-      while (entries.hasMoreElements()) {
-        JarEntry entry = entries.nextElement();
-        Matcher m = libJarPattern.matcher(entry.getName());
-        if (m.matches()) {
-          File file = new File(parentDirStr, "." + pathPrefix + "."
-            + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
-          try (FileOutputStream outStream = new FileOutputStream(file)) {
-            IOUtils.copyBytes(jarFile.getInputStream(entry),
-              outStream, conf, true);
+    FileSystem fs = pathPattern.getFileSystem(conf);
+    Path pathPattern1 = fs.isDirectory(pathPattern) ?
+      new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a directory is
specified
+    FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all files that match
the pattern
+    if (fileStatuses == null || fileStatuses.length == 0) {  // if no one matches
+      throw new FileNotFoundException(pathPattern1.toString());
+    } else {
+      boolean validFileEncountered = false;
+      for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file that match
the pattern
+        if (fs.isFile(path)) {  // only process files, skip for directories
+          File dst = new File(parentDirStr, "." + pathPrefix + "."
+            + path.getName() + "." + System.currentTimeMillis() + ".jar");
+          fs.copyToLocalFile(path, new Path(dst.toString()));
+          dst.deleteOnExit();
+
+          addURL(dst.getCanonicalFile().toURI().toURL());
+
+          JarFile jarFile = new JarFile(dst.toString());
+          try {
+            Enumeration<JarEntry> entries = jarFile.entries();  // get entries inside
a jar file
+            while (entries.hasMoreElements()) {
+              JarEntry entry = entries.nextElement();
+              Matcher m = libJarPattern.matcher(entry.getName());
+              if (m.matches()) {
+                File file = new File(parentDirStr, "." + pathPrefix + "."
+                  + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+                try (FileOutputStream outStream = new FileOutputStream(file)) {
+                  IOUtils.copyBytes(jarFile.getInputStream(entry),
+                    outStream, conf, true);
+                }
+                file.deleteOnExit();
+                addURL(file.toURI().toURL());
+              }
+            }
+          } finally {
+            jarFile.close();
           }
-          file.deleteOnExit();
-          addURL(file.toURI().toURL());
+
+          validFileEncountered = true;  // Set to true when encountering a file
         }
       }
-    } finally {
-      jarFile.close();
+      if (validFileEncountered == false) {  // all items returned by globStatus() are directories
+        throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+      }
     }
   }
 
@@ -227,7 +247,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       return cl;
     }
 
-    if (!pathStr.endsWith(".jar")) {
+    if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
       throw new IOException(pathStr + ": not a jar file?");
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
index daba459..967d272 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -109,4 +110,46 @@ public class TestCoprocessorClassLoader {
     }
     fail("Could not find the expected lib jar file");
   }
+
+  // HBASE-14548
+  @Test
+  public void testDirectoryAndWildcard() throws Exception {
+    String testClassName = "TestClass";
+    String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+    System.out.println(dataTestDir);
+    String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+    ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+    ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+    CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+    CoprocessorClassLoader coprocessorClassLoader = null;
+    Path testPath = null;
+
+    // Directory
+    testPath = new Path(localDirContainingJar);
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1",
conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.jar
+    testPath = new Path(localDirContainingJar, "*.jar");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2",
conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.j*
+    testPath = new Path(localDirContainingJar, "*.j*");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3",
conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+  }
+
+  /**
+   * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+   * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+   * @param className the expected class to be loaded by the coprocessorClassLoader
+   * @throws ClassNotFoundException
+   */
+  private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader,
String className)
+    throws ClassNotFoundException{
+    assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+    assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/src/main/asciidoc/_chapters/cp.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/cp.adoc b/src/main/asciidoc/_chapters/cp.adoc
index 793bb6f..376899a 100644
--- a/src/main/asciidoc/_chapters/cp.adoc
+++ b/src/main/asciidoc/_chapters/cp.adoc
@@ -320,7 +320,14 @@ The value contains four pieces of information which are separated by
the pipe (`
 * File path: The jar file containing the Coprocessor implementation must be in a location
where
 all region servers can read it. +
 You could copy the file onto the local disk on each region server, but it is recommended
to store
-it in HDFS.
+it in HDFS. +
+https://issues.apache.org/jira/browse/HBASE-14548[HBASE-14548] allows a directory containing
the jars
+or some wildcards to be specified, such as: hdfs://<namenode>:<port>/user/<hadoop-user>/
or
+hdfs://<namenode>:<port>/user/<hadoop-user>/*.jar. Please note that if
a directory is specified,
+all jar files(.jar) directly in the directory are added,
+but it does not search files in the subtree rooted in the directory.
+And do not contain any wildcard if you would like to specify a directory.
+This enhancement applies to the ways of using the JAVA API as well.
 * Class name: The full class name of the Coprocessor.
 * Priority: An integer. The framework will determine the execution sequence of all configured
 observers registered at the same hook using priorities. This field can be left blank. In
that


Mime
View raw message