beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhalp...@apache.org
Subject [1/2] beam git commit: [BEAM-1676] Internalize ClassPath from Guava 21.0 to fix the failures related to having whitespaces in resource names.
Date Mon, 01 May 2017 21:23:57 GMT
Repository: beam
Updated Branches:
  refs/heads/master 3ae944130 -> c2a555783


[BEAM-1676] Internalize ClassPath from Guava 21.0 to fix the failures related to having whitespaces
in resource names.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0788d3db
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0788d3db
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0788d3db

Branch: refs/heads/master
Commit: 0788d3db82f7de3f749a279274208186ad928f3d
Parents: 3ae9441
Author: Stas Levin <staslevin@apache.org>
Authored: Sun Apr 30 13:56:40 2017 +0300
Committer: Dan Halperin <dhalperi@google.com>
Committed: Mon May 1 14:23:45 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/beam/findbugs-filter.xml |   6 +
 .../org/apache/beam/sdk/util/ApiSurface.java    |   4 +-
 .../org/apache/beam/sdk/util/ClassPath.java     | 544 +++++++++++++++++++
 3 files changed, 551 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/0788d3db/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
----------------------------------------------------------------------
diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index d03fbf3..d1d8b4d 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -379,6 +379,12 @@
     <!--[BEAM-421] Class doesn't override equals in superclass-->
   </Match>
   <Match>
+    <Class name="org.apache.beam.sdk.util.ClassPath$ClassInfo"/>
+    <Method name="equals"/>
+    <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS"/>
+    <!--[BEAM-1676] Class doesn't override equals in superclass-->
+  </Match>
+  <Match>
     <Class name="org.apache.beam.sdk.util.AutoValue_GcsUtil_StorageObjectOrIOException"/>
     <Bug pattern="NM_CLASS_NOT_EXCEPTION"/>
     <!-- It is clear from the name that this class holds either StorageObject or IOException.
-->

http://git-wip-us.apache.org/repos/asf/beam/blob/0788d3db/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
index 9530e88..8c2b988 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
@@ -33,8 +33,6 @@ import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
-import com.google.common.reflect.ClassPath;
-import com.google.common.reflect.ClassPath.ClassInfo;
 import com.google.common.reflect.Invokable;
 import com.google.common.reflect.Parameter;
 import com.google.common.reflect.TypeToken;
@@ -353,7 +351,7 @@ public class ApiSurface {
     ClassPath classPath = ClassPath.from(classLoader);
 
     Set<Class<?>> newRootClasses = Sets.newHashSet();
-    for (ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName)) {
+    for (ClassPath.ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName))
{
       Class clazz = classInfo.load();
       if (exposed(clazz.getModifiers())) {
         newRootClasses.add(clazz);

http://git-wip-us.apache.org/repos/asf/beam/blob/0788d3db/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
new file mode 100644
index 0000000..35632ed
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.annotations.Beta;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Predicate;
+import com.google.common.base.Splitter;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MultimapBuilder;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteSource;
+import com.google.common.io.CharSource;
+import com.google.common.io.Resources;
+import com.google.common.reflect.Reflection;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.Charset;
+import java.util.Enumeration;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Scans the source of a {@link ClassLoader} and finds all loadable classes and resources.
+ *
+ * <p><b>Warning:</b> Currently only {@link URLClassLoader} and only {@code
file://} urls are
+ * supported.
+ * </p>
+ *
+ * <p>Based on Ben Yu's implementation in
+ * <a href="https://github.com/google/guava/blob/896c51abd32e136621c13d56b6130d0a72f4957a/guava/src/com/google/common/reflect/ClassPath.java">Guava</a>.
+ * </p>
+ *
+ * <p><b>Note:</b> Internalised here to avoid a forced upgrade to
+ * <a href="https://github.com/google/guava/releases/tag/v21.0">Guava 21.0 which requires
+ * Java 8.</a>
+ * </p>
+ */
+@Beta
+final class ClassPath {
+
+  private static final Logger logger = LoggerFactory.getLogger(ClassPath.class.getName());
+
+  private static final Predicate<ClassInfo> IS_TOP_LEVEL =
+      new Predicate<ClassInfo>() {
+
+        @Override
+        public boolean apply(ClassInfo info) {
+          return info != null && info.className.indexOf('$') == -1;
+        }
+      };
+
+  /** Separator for the Class-Path manifest attribute value in jar files. */
+  private static final Splitter CLASS_PATH_ATTRIBUTE_SEPARATOR =
+      Splitter.on(" ").omitEmptyStrings();
+
+  private static final String CLASS_FILE_NAME_EXTENSION = ".class";
+
+  private final ImmutableSet<ResourceInfo> resources;
+
+  private ClassPath(ImmutableSet<ResourceInfo> resources) {
+    this.resources = resources;
+  }
+
+  /**
+   * Returns a {@code ClassPath} representing all classes and resources loadable from {@code
+   * classloader} and its parent class loaders.
+   *
+   * <p><b>Warning:</b> Currently only {@link URLClassLoader} and only
{@code file://} urls are
+   * supported.
+   *
+   * @throws IOException if the attempt to read class path resources (jar files or directories)
+   *     failed.
+   */
+  public static ClassPath from(ClassLoader classloader) throws IOException {
+    DefaultScanner scanner = new DefaultScanner();
+    scanner.scan(classloader);
+    return new ClassPath(scanner.getResources());
+  }
+
+  /**
+   * Returns all resources loadable from the current class path, including the class files
of all
+   * loadable classes but excluding the "META-INF/MANIFEST.MF" file.
+   */
+  public ImmutableSet<ResourceInfo> getResources() {
+    return resources;
+  }
+
+  /**
+   * Returns all classes loadable from the current class path.
+   *
+   * @since 16.0
+   */
+  public ImmutableSet<ClassInfo> getAllClasses() {
+    return FluentIterable.from(resources).filter(ClassInfo.class).toSet();
+  }
+
+  /** Returns all top level classes loadable from the current class path. */
+  public ImmutableSet<ClassInfo> getTopLevelClasses() {
+    return FluentIterable.from(resources).filter(ClassInfo.class).filter(IS_TOP_LEVEL).toSet();
+  }
+
+  /** Returns all top level classes whose package name is {@code packageName}. */
+  public ImmutableSet<ClassInfo> getTopLevelClasses(String packageName) {
+    checkNotNull(packageName);
+    ImmutableSet.Builder<ClassInfo> builder = ImmutableSet.builder();
+    for (ClassInfo classInfo : getTopLevelClasses()) {
+      if (classInfo.getPackageName().equals(packageName)) {
+        builder.add(classInfo);
+      }
+    }
+    return builder.build();
+  }
+
+  /**
+   * Returns all top level classes whose package name is {@code packageName} or starts with
+   * {@code packageName} followed by a '.'.
+   */
+  public ImmutableSet<ClassInfo> getTopLevelClassesRecursive(String packageName) {
+    checkNotNull(packageName);
+    String packagePrefix = packageName + '.';
+    ImmutableSet.Builder<ClassInfo> builder = ImmutableSet.builder();
+    for (ClassInfo classInfo : getTopLevelClasses()) {
+      if (classInfo.getName().startsWith(packagePrefix)) {
+        builder.add(classInfo);
+      }
+    }
+    return builder.build();
+  }
+
+  /**
+   * Represents a class path resource that can be either a class file or any other resource
file
+   * loadable from the class path.
+   *
+   * @since 14.0
+   */
+  @Beta
+  public static class ResourceInfo {
+
+    private final String resourceName;
+
+    final ClassLoader loader;
+
+    static ResourceInfo of(String resourceName, ClassLoader loader) {
+      if (resourceName.endsWith(CLASS_FILE_NAME_EXTENSION)) {
+        return new ClassInfo(resourceName, loader);
+      } else {
+        return new ResourceInfo(resourceName, loader);
+      }
+    }
+
+    ResourceInfo(String resourceName, ClassLoader loader) {
+      this.resourceName = checkNotNull(resourceName);
+      this.loader = checkNotNull(loader);
+    }
+
+    /**
+     * Returns the url identifying the resource.
+     *
+     * <p>See {@link ClassLoader#getResource}
+     *
+     * @throws NoSuchElementException if the resource cannot be loaded through the class
loader,
+     *     despite physically existing in the class path.
+     */
+    public final URL url() {
+      URL url = loader.getResource(resourceName);
+      if (url == null) {
+        throw new NoSuchElementException(resourceName);
+      }
+      return url;
+    }
+
+    /**
+     * Returns a {@link ByteSource} view of the resource from which its bytes can be read.
+     *
+     * @throws NoSuchElementException if the resource cannot be loaded through the class
loader,
+     *     despite physically existing in the class path.
+     * @since 20.0
+     */
+    public final ByteSource asByteSource() {
+      return Resources.asByteSource(url());
+    }
+
+    /**
+     * Returns a {@link CharSource} view of the resource from which its bytes can be read
as
+     * characters decoded with the given {@code charset}.
+     *
+     * @throws NoSuchElementException if the resource cannot be loaded through the class
loader,
+     *     despite physically existing in the class path.
+     * @since 20.0
+     */
+    public final CharSource asCharSource(Charset charset) {
+      return Resources.asCharSource(url(), charset);
+    }
+
+    /** Returns the fully qualified name of the resource. Such as "com/mycomp/foo/bar.txt".
*/
+    public final String getResourceName() {
+      return resourceName;
+    }
+
+    @Override
+    public int hashCode() {
+      return resourceName.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj instanceof ResourceInfo) {
+        ResourceInfo that = (ResourceInfo) obj;
+        return resourceName.equals(that.resourceName) && loader == that.loader;
+      }
+      return false;
+    }
+
+    // Do not change this arbitrarily. We rely on it for sorting ResourceInfo.
+    @Override
+    public String toString() {
+      return resourceName;
+    }
+  }
+
+  /**
+   * Represents a class that can be loaded through {@link #load}.
+   *
+   * @since 14.0
+   */
+  @Beta
+  static final class ClassInfo extends ResourceInfo {
+
+    private final String className;
+
+    ClassInfo(String resourceName, ClassLoader loader) {
+      super(resourceName, loader);
+      this.className = getClassName(resourceName);
+    }
+
+    /**
+     * Returns the package name of the class, without attempting to load the class.
+     *
+     * <p>Behaves identically to {@link Package#getName()} but does not require the
class (or
+     * package) to be loaded.
+     */
+    public String getPackageName() {
+      return Reflection.getPackageName(className);
+    }
+
+    /**
+     * Returns the simple name of the underlying class as given in the source code.
+     *
+     * <p>Behaves identically to {@link Class#getSimpleName()} but does not require
the class
+     * to be
+     * loaded.
+     */
+    public String getSimpleName() {
+      int lastDollarSign = className.lastIndexOf('$');
+      if (lastDollarSign != -1) {
+        String innerClassName = className.substring(lastDollarSign + 1);
+        // local and anonymous classes are prefixed with number (1,2,3...), anonymous classes
are
+        // entirely numeric whereas local classes have the user supplied name as a suffix
+        return CharMatcher.digit().trimLeadingFrom(innerClassName);
+      }
+      String packageName = getPackageName();
+      if (packageName.isEmpty()) {
+        return className;
+      }
+
+      // Since this is a top level class, its simple name is always the part after package
name.
+      return className.substring(packageName.length() + 1);
+    }
+
+    /**
+     * Returns the fully qualified name of the class.
+     *
+     * <p>Behaves identically to {@link Class#getName()} but does not require the class
to be
+     * loaded.
+     */
+    public String getName() {
+      return className;
+    }
+
+    /**
+     * Loads (but doesn't link or initialize) the class.
+     *
+     * @throws LinkageError when there were errors in loading classes that this class depends
on.
+     *     For example, {@link NoClassDefFoundError}.
+     */
+    public Class<?> load() {
+      try {
+        return loader.loadClass(className);
+      } catch (ClassNotFoundException e) {
+        // Shouldn't happen, since the class name is read from the class path.
+        throw new IllegalStateException(e);
+      }
+    }
+
+    @Override
+    public String toString() {
+      return className;
+    }
+  }
+
+  /**
+   * Abstract class that scans through the class path represented by a {@link ClassLoader}
and
+   * calls
+   * {@link #scanDirectory} and {@link #scanJarFile} for directories and jar files on the
class
+   * path
+   * respectively.
+   */
+  abstract static class Scanner {
+
+    // We only scan each file once independent of the classloader that resource might be
+    // associated
+    // with.
+    private final Set<File> scannedUris = Sets.newHashSet();
+
+    public final void scan(ClassLoader classloader) throws IOException {
+      for (Map.Entry<File, ClassLoader> entry : getClassPathEntries(classloader).entrySet())
{
+        scan(entry.getKey(), entry.getValue());
+      }
+    }
+
+    /** Called when a directory is scanned for resource files. */
+    protected abstract void scanDirectory(ClassLoader loader, File directory) throws IOException;
+
+    /** Called when a jar file is scanned for resource entries. */
+    protected abstract void scanJarFile(ClassLoader loader, JarFile file) throws IOException;
+
+    @VisibleForTesting
+    final void scan(File file, ClassLoader classloader) throws IOException {
+      if (scannedUris.add(file.getCanonicalFile())) {
+        scanFrom(file, classloader);
+      }
+    }
+
+    private void scanFrom(File file, ClassLoader classloader) throws IOException {
+      try {
+        if (!file.exists()) {
+          return;
+        }
+      } catch (SecurityException e) {
+        logger.warn("Cannot access " + file + ": " + e);
+        return;
+      }
+      if (file.isDirectory()) {
+        scanDirectory(classloader, file);
+      } else {
+        scanJar(file, classloader);
+      }
+    }
+
+    private void scanJar(File file, ClassLoader classloader) throws IOException {
+      JarFile jarFile;
+      try {
+        jarFile = new JarFile(file);
+      } catch (IOException e) {
+        // Not a jar file
+        return;
+      }
+      try {
+        for (File path : getClassPathFromManifest(file, jarFile.getManifest())) {
+          scan(path, classloader);
+        }
+        scanJarFile(classloader, jarFile);
+      } finally {
+        try {
+          jarFile.close();
+        } catch (IOException ignored) {
+        }
+      }
+    }
+
+    /**
+     * Returns the class path URIs specified by the {@code Class-Path} manifest attribute,
+     * according
+     * to
+     * <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Main_Attributes">
+     * JAR File Specification</a>. If {@code manifest} is null, it means the jar file
has no
+     * manifest, and an empty set will be returned.
+     */
+    @VisibleForTesting
+    static ImmutableSet<File> getClassPathFromManifest(File jarFile,
+                                                       @Nullable Manifest manifest) {
+      if (manifest == null) {
+        return ImmutableSet.of();
+      }
+      ImmutableSet.Builder<File> builder = ImmutableSet.builder();
+      String classpathAttribute =
+          manifest.getMainAttributes().getValue(Attributes.Name.CLASS_PATH.toString());
+      if (classpathAttribute != null) {
+        for (String path : CLASS_PATH_ATTRIBUTE_SEPARATOR.split(classpathAttribute)) {
+          URL url;
+          try {
+            url = getClassPathEntry(jarFile, path);
+          } catch (MalformedURLException e) {
+            // Ignore bad entry
+            logger.warn("Invalid Class-Path entry: " + path);
+            continue;
+          }
+          if (url.getProtocol().equals("file")) {
+            builder.add(toFile(url));
+          }
+        }
+      }
+      return builder.build();
+    }
+
+    @VisibleForTesting
+    static ImmutableMap<File, ClassLoader> getClassPathEntries(ClassLoader classloader)
{
+      LinkedHashMap<File, ClassLoader> entries = Maps.newLinkedHashMap();
+      // Search parent first, since it's the order ClassLoader#loadClass() uses.
+      ClassLoader parent = classloader.getParent();
+      if (parent != null) {
+        entries.putAll(getClassPathEntries(parent));
+      }
+      if (classloader instanceof URLClassLoader) {
+        URLClassLoader urlClassLoader = (URLClassLoader) classloader;
+        for (URL entry : urlClassLoader.getURLs()) {
+          if (entry.getProtocol().equals("file")) {
+            File file = toFile(entry);
+            if (!entries.containsKey(file)) {
+              entries.put(file, classloader);
+            }
+          }
+        }
+      }
+      return ImmutableMap.copyOf(entries);
+    }
+
+    /**
+     * Returns the absolute uri of the Class-Path entry value as specified in
+     * <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Main_Attributes">
+     * JAR File Specification</a>. Even though the specification only talks about relative
urls,
+     * absolute urls are actually supported too (for example, in Maven surefire plugin).
+     */
+    @VisibleForTesting
+    static URL getClassPathEntry(File jarFile, String path) throws MalformedURLException
{
+      return new URL(jarFile.toURI().toURL(), path);
+    }
+  }
+
+  @VisibleForTesting
+  static final class DefaultScanner extends Scanner {
+
+    private final SetMultimap<ClassLoader, String> resources =
+        MultimapBuilder.hashKeys().linkedHashSetValues().build();
+
+    ImmutableSet<ResourceInfo> getResources() {
+      ImmutableSet.Builder<ResourceInfo> builder = ImmutableSet.builder();
+      for (Map.Entry<ClassLoader, String> entry : resources.entries()) {
+        builder.add(ResourceInfo.of(entry.getValue(), entry.getKey()));
+      }
+      return builder.build();
+    }
+
+    @Override
+    protected void scanJarFile(ClassLoader classloader, JarFile file) {
+      Enumeration<JarEntry> entries = file.entries();
+      while (entries.hasMoreElements()) {
+        JarEntry entry = entries.nextElement();
+        if (entry.isDirectory() || entry.getName().equals(JarFile.MANIFEST_NAME)) {
+          continue;
+        }
+        resources.get(classloader).add(entry.getName());
+      }
+    }
+
+    @Override
+    protected void scanDirectory(ClassLoader classloader, File directory) throws IOException
{
+      scanDirectory(directory, classloader, "");
+    }
+
+    private void scanDirectory(File directory, ClassLoader classloader, String packagePrefix)
+        throws IOException {
+      File[] files = directory.listFiles();
+      if (files == null) {
+        logger.warn("Cannot read directory " + directory);
+        // IO error, just skip the directory
+        return;
+      }
+      for (File f : files) {
+        String name = f.getName();
+        if (f.isDirectory()) {
+          scanDirectory(f, classloader, packagePrefix + name + "/");
+        } else {
+          String resourceName = packagePrefix + name;
+          if (!resourceName.equals(JarFile.MANIFEST_NAME)) {
+            resources.get(classloader).add(resourceName);
+          }
+        }
+      }
+    }
+  }
+
+  @VisibleForTesting
+  static String getClassName(String filename) {
+    int classNameEnd = filename.length() - CLASS_FILE_NAME_EXTENSION.length();
+    return filename.substring(0, classNameEnd).replace('/', '.');
+  }
+
+  @VisibleForTesting
+  static File toFile(URL url) {
+    checkArgument(url.getProtocol().equals("file"));
+    try {
+      return new File(url.toURI());  // Accepts escaped characters like %20.
+    } catch (URISyntaxException e) {  // URL.toURI() doesn't escape chars.
+      return new File(url.getPath());  // Accepts non-escaped chars like space.
+    }
+  }
+}


Mime
View raw message