ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [1/2] ignite git commit: Finalization.
Date Thu, 15 Sep 2016 16:32:38 GMT
Repository: ignite
Updated Branches:
  refs/heads/ignite-3909 5fa1e6068 -> a6ed71b2b


Finalization.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a68826dd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a68826dd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a68826dd

Branch: refs/heads/ignite-3909
Commit: a68826dd9fb0383e61f17bb72e26c90c3856bbca
Parents: 5fa1e60
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Thu Sep 15 19:14:02 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Thu Sep 15 19:14:02 2016 +0300

----------------------------------------------------------------------
 .../src/main/java/com/mapr/DummyFileSystem.java | 99 ++++++++++++++++++++
 .../processors/hadoop/HadoopClassLoader.java    | 71 ++++++++++----
 2 files changed, 150 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a68826dd/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java b/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java
new file mode 100644
index 0000000..63312c8
--- /dev/null
+++ b/modules/hadoop/src/main/java/com/mapr/DummyFileSystem.java
@@ -0,0 +1,99 @@
+package com.mapr;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.URI;
+
+/**
+ * Created by vozerov on 9/15/2016.
+ */
+public class DummyFileSystem extends FileSystem {
+
+    public DummyFileSystem() throws IOException {
+        try {
+            Method m =  UserGroupInformation.class.getDeclaredMethod("reset");
+
+            m.setAccessible(true);
+
+            m.invoke(null);
+
+            Field f = Groups.class.getDeclaredField("GROUPS");
+
+            f.setAccessible(true);
+
+            f.set(null, null);
+        }
+        catch (Exception ignore) {
+            throw new RuntimeException(ignore);
+        }
+
+        UserGroupInformation.getLoginUser();
+    }
+
+    @Override
+    public URI getUri() {
+        return null;
+    }
+
+    @Override
+    public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+        return null;
+    }
+
+    @Override
+    public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite,
int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
{
+        return null;
+    }
+
+    @Override
+    public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws
IOException {
+        return null;
+    }
+
+    @Override
+    public boolean rename(Path src, Path dst) throws IOException {
+        return false;
+    }
+
+    @Override
+    public boolean delete(Path f, boolean recursive) throws IOException {
+        return false;
+    }
+
+    @Override
+    public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
+        return new FileStatus[0];
+    }
+
+    @Override
+    public void setWorkingDirectory(Path new_dir) {
+
+    }
+
+    @Override
+    public Path getWorkingDirectory() {
+        return null;
+    }
+
+    @Override
+    public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+        return false;
+    }
+
+    @Override
+    public FileStatus getFileStatus(Path f) throws IOException {
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/a68826dd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index c0b671e..f457ffa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -410,43 +410,74 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
 
         String parentCls = clsName.substring(0, idx);
 
-        // TODO: Looks suspicious.
-        if (ctx.visited.contains(parentCls))
-            return false;
-
         return hasExternalDependencies(parentCls, ctx);
     }
 
     /**
+     * Check whether class name starts with particular prefix.
+     *
+     * @param cls Class name.
+     * @param skip Prefix to skip.
+     * @param prefix Prefix.
+     * @return {@code True} if starts with.
+     */
+    private static boolean startsWith(String cls, String skip, String prefix) {
+        assert cls.startsWith(skip);
+
+        return cls.startsWith(prefix, skip.length());
+    }
+
+    private static final String PKG_ORG = "org.";
+
+    private static final String PKG_ORG_APACHE = "org.apache.";
+
+    /**
      * Whether we know in advance whether class has dependency or not.
      *
      * @param cls Class.
      * @return Result.
      */
     @Nullable private static Boolean hasDependencyPredefined(String cls) {
-        // 1. Java systm classes never has dependencies.
-        if (cls.startsWith("java.") || cls.startsWith("javax.") || cls.startsWith("sun.")
|| cls.startsWith("com.sun."))
-            return false;
-
-        // 2. Some other well-known packages.
-        if (cls.startsWith("org.jsr166.") ||  cls.startsWith("org.w3c.") || cls.startsWith("org.xml.sax.")
|| cls.startsWith("org.slf4j.") || cls.startsWith("com.google.common."))
-            return false;
+        // Large "org" group.
+        if (cls.startsWith(PKG_ORG)) {
+            // Large "apache" group.
+            if (cls.startsWith(PKG_ORG_APACHE)) {
+                // Hadoop classes always have dependencies.
+                if (startsWith(cls, PKG_ORG_APACHE, "hadoop."))
+                    return true;
+
+                // Filter out Ignite classes which definitely do not have dependencies.
+                if (startsWith(cls, PKG_ORG_APACHE, "ignite.")) {
+                    if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") &&
!cls.contains(".fs."))
+                        return false;
+                }
 
-        // 3. Special handling for "org.apache"
-        if (cls.startsWith("org.apache.")) {
-            if (cls.startsWith("org.apache.ignite")) {
-                if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") &&
!cls.contains(".fs."))
+                // Other well-known "org.apache" packages.
+                if (startsWith(cls, PKG_ORG_APACHE, "commons.") ||
+                    startsWith(cls, PKG_ORG_APACHE, "log4j.") ||
+                    startsWith(cls, PKG_ORG_APACHE, "xerces."))
                     return false;
             }
 
-            if (cls.startsWith("org.apache.hadoop"))
-                return true;
-
-            if (cls.startsWith("org.apache.xerces") || cls.startsWith("org.apache.log4j")
|| cls.startsWith("org.apache.commons.logging") || cls.startsWith("org.apache.commons.lang")
|| cls.startsWith("org.apache.commons.collections")
-                || cls.startsWith("org.apache.commons.configuration"))
+            // Other well-known "org" packages.
+            if (startsWith(cls, PKG_ORG, "jsr166.") ||
+                startsWith(cls, PKG_ORG, "w3c.") ||
+                startsWith(cls, PKG_ORG, "slf4j.") ||
+                startsWith(cls, PKG_ORG, "xml.sax."))
                 return false;
         }
 
+        // Filter out Java system packages.
+        if (cls.startsWith("java.") ||
+            cls.startsWith("javax.") ||
+            cls.startsWith("sun.") ||
+            cls.startsWith("com.sun."))
+            return false;
+
+        // Other well-known packages.
+        if (cls.startsWith("com.google.common"))
+            return false;
+
         // No more guesses, will parse the class.
         return null;
     }


Mime
View raw message