ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [1/9] ignite git commit: IGNITE-3929: wip.
Date Tue, 20 Sep 2016 13:55:10 GMT
Repository: ignite
Updated Branches:
  refs/heads/ignite-3929-1 [created] c9e665e22


IGNITE-3929: wip.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/df3ad76f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/df3ad76f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/df3ad76f

Branch: refs/heads/ignite-3929-1
Commit: df3ad76f3f105fbda34ae5bb79463ab41c46fe85
Parents: 857cdcd
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Mon Sep 19 21:27:55 2016 +0300
Committer: iveselovskiy <iveselovskiy@gridgain.com>
Committed: Mon Sep 19 21:27:55 2016 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopSnappyTest.java     |   6 +-
 .../processors/hadoop/HadoopClassLoader.java    | 225 +++++++++++++++----
 .../hadoop/HadoopClassLoaderUtils.java          |   2 +-
 .../internal/processors/hadoop/LoadHelper.java  |  42 ++++
 .../ignite/internal/processors/hadoop/XXX.java  |   7 +
 5 files changed, 234 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/df3ad76f/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
index b4e3dc2..0b86c50 100644
--- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
+++ b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
@@ -45,10 +45,10 @@ public class HadoopSnappyTest extends GridCommonAbstractTest {
      */
     public void testSnappy() throws Throwable {
         // Run Snappy test in default class loader:
-        checkSnappy();
+        //checkSnappy();
 
         // Run the same in several more class loaders simulating jobs and tasks:
-        for (int i = 0; i < 2; i++) {
+        for (int i = 0; i < 3; i++) {
             ClassLoader hadoopClsLdr = new HadoopClassLoader(null, "cl-" + i, null);
 
             Class<?> cls = (Class)Class.forName(HadoopSnappyTest.class.getName(), true,
hadoopClsLdr);
@@ -57,6 +57,8 @@ public class HadoopSnappyTest extends GridCommonAbstractTest {
 
             U.invoke(cls, null, "checkSnappy");
         }
+
+        //checkSnappy();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/df3ad76f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 30a6e72..c6d2296 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -17,7 +17,10 @@
 
 package org.apache.ignite.internal.processors.hadoop;
 
+import java.util.Collections;
+import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.util.ClassCache;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -61,8 +64,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
     public static final String CLS_DAEMON_REPLACE = "org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon";
 
     /** Hadoop class name: ShutdownHookManager replacement. */
-    public static final String CLS_SHUTDOWN_HOOK_MANAGER_REPLACE =
-        "org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager";
+    public static final String CLS_SHUTDOWN_HOOK_MANAGER_REPLACE = "org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager";
 
     /** Name of libhadoop library. */
     private static final String LIBHADOOP = "hadoop.";
@@ -83,14 +85,14 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
     private final ConcurrentMap<String, Class> cacheMap = new ConcurrentHashMap<>();
 
     /** Diagnostic name of this class loader. */
-    @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
-    private final String name;
+    @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) private final String name;
 
     /** Native library names. */
-    private final String[] libNames;
+    private String[] loadedLibNames;
 
     /**
      * Gets name for Job class loader. The name is specific for local node id.
+     *
      * @param locNodeId The local node id.
      * @return The class loader name.
      */
@@ -100,6 +102,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
 
     /**
      * Gets name for the task class loader. Task class loader
+     *
      * @param info The task info.
      * @param prefix Get only prefix (without task type and number)
      * @return The class loader name.
@@ -122,70 +125,202 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
         super(addHadoopUrls(urls), APP_CLS_LDR);
 
         assert !(getParent() instanceof HadoopClassLoader);
+        assert getClass().getClassLoader() == APP_CLS_LDR; // by definition, app cls loader
created in such way.
 
         this.name = name;
-        this.libNames = libNames;
 
-        initializeNativeLibraries();
+        // TODO: for POC:
+        if (libNames == null)
+            libNames = new String[] { "hadoop" };
+
+        setNativeLibrariesToBeInjectedIfNeeded(this, libNames);
     }
 
+    //    /**
+    //     * Workaround to load native Hadoop libraries. Java doesn't allow native libraries
to be loaded from different
+    //     * classloaders. But we load Hadoop classes many times and one of these classes
- {@code NativeCodeLoader} - tries
+
+    //     * to load the same native library over and over again.
+    //     * <p>
+    //     * To fix the problem, we force native library load in parent class loader and
then "link" handle to this native
+    //     * library to our class loader. As a result, our class loader will think that the
library is already loaded and will
+    //     * be able to link native methods.
+    //     *
+    //     * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/invocation.html#library_version">
+    //     *     JNI specification</a>
+    //     */
+    //    private void initializeNativeLibraries() {
+    //        try {
+    //            // This must trigger native library load.
+    //            // TODO: Do not delegate to APP LDR
+    //            Class.forName(CLS_NATIVE_CODE_LOADER, true, this);
+    //
+    //            final Vector<Object> curVector = U.field(this, "nativeLibraries");
+    //
+    //            // TODO: Do not delegate to APP LDR
+    //            ClassLoader ldr = APP_CLS_LDR;
+    //
+    //            while (ldr != null) {
+    //                Vector vector = U.field(ldr, "nativeLibraries");
+    //
+    //                for (Object lib : vector) {
+    //                    String name = U.field(lib, "name");
+    //
+    //                    boolean add = name.contains(LIBHADOOP);
+    //
+    //                    if (!add && libNames != null) {
+    //                        for (String libName : libNames) {
+    //                            if (libName != null && name.contains(libName))
{
+    //                                add = true;
+    //
+    //                                break;
+    //                            }
+    //                        }
+    //                    }
+    //
+    //                    if (add) {
+    //                        curVector.add(lib);
+    //
+    //                        return;
+    //                    }
+    //                }
+    //
+    //                ldr = ldr.getParent();
+    //            }
+    //        }
+    //        catch (Exception e) {
+    //            U.quietAndWarn(null, "Failed to initialize Hadoop native library " +
+    //                "(native Hadoop methods might not work properly): " + e);
+    //        }
+    //    }
+
+    /** */
+    private static volatile Collection<Object> nativeLibrariesToBeInjected;
+
     /**
-     * Workaround to load native Hadoop libraries. Java doesn't allow native libraries to
be loaded from different
-     * classloaders. But we load Hadoop classes many times and one of these classes - {@code
NativeCodeLoader} - tries
-     * to load the same native library over and over again.
-     * <p>
-     * To fix the problem, we force native library load in parent class loader and then "link"
handle to this native
-     * library to our class loader. As a result, our class loader will think that the library
is already loaded and will
-     * be able to link native methods.
-     *
-     * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/invocation.html#library_version">
-     *     JNI specification</a>
+     * This method will be invoked for each created instance of HadoopClassLoader, but the
list of native libraries will
+     * be loaded only once.
      */
-    private void initializeNativeLibraries() {
-        try {
-            // This must trigger native library load.
-            // TODO: Do not delegate to APP LDR
-            Class.forName(CLS_NATIVE_CODE_LOADER, true, APP_CLS_LDR);
+    private static void setNativeLibrariesToBeInjectedIfNeeded(HadoopClassLoader instance,
String[] libs) {
+        if (libs == null)
+            return;
+
+        boolean created = false;
+
+        // 1. If needed, init the native lib data collection:
+        if (nativeLibrariesToBeInjected == null) {
+            synchronized (HadoopClassLoader.class) {
+                if (nativeLibrariesToBeInjected == null) {
+                    instance.runLoadingCode(libs);
+
+                    nativeLibrariesToBeInjected = instance.collectNativeLibraries();
+
+                    created = true;
+                }
+            }
+        }
 
+        assert nativeLibrariesToBeInjected != null;
+
+        // 2. Inject libraries:
+        if (!created)
+            // This is an instance that did not load the libs:
+            instance.injectNatives();
+    }
+
+    /**
+     * Injects previously
+     */
+    private void injectNatives() {
+        try {
+            // 2. Init this instance with the natives:
             final Vector<Object> curVector = U.field(this, "nativeLibraries");
 
-            // TODO: Do not delegate to APP LDR
-            ClassLoader ldr = APP_CLS_LDR;
+            curVector.addAll(nativeLibrariesToBeInjected);
+        }
+        catch (Exception e) {
+            U.quietAndWarn(null, "Failed to initialize Hadoop native library " +
+                 "(native Hadoop methods might not work properly): " + e);
+        }
+    }
 
-            while (ldr != null) {
-                Vector vector = U.field(ldr, "nativeLibraries");
+    /**
+     *
+     * @return
+     */
+    private Collection<Object> collectNativeLibraries() {
+        List<Object> target = new ArrayList<>();
 
-                for (Object lib : vector) {
-                    String name = U.field(lib, "name");
+        ClassLoader ldr = this;
 
-                    boolean add = name.contains(LIBHADOOP);
+        while (ldr != null) {
+            collectNativeLibrariesFromLoader(ldr, target);
 
-                    if (!add && libNames != null) {
-                        for (String libName : libNames) {
-                            if (libName != null && name.contains(libName)) {
-                                add = true;
+            ldr = ldr.getParent();
+        }
 
-                                break;
-                            }
-                        }
-                    }
+        return Collections.unmodifiableList(target);
+    }
 
-                    if (add) {
-                        curVector.add(lib);
+    /**
+     * Run default or user code to force native libs loading:
+     */
+    private void runLoadingCode(String[] libs) {
+        try {
+            // TODO: "XXX" is a special class loaded by Hadoop class loader (simulating Hadoop
class).
+            // NB: this sample class must *not* cause loading of any natives.
+            Class<?> sampleCls = this.loadClass(XXX.class.getName(), true);
 
-                        return;
-                    }
-                }
+            assert sampleCls != null;
+            assert sampleCls.getClassLoader() == this;
+
+            Collection<String> loadedLibs = new ArrayList<>();
 
-                ldr = ldr.getParent();
+            for (String lib: libs) {
+                boolean ok = LoadHelper.tryLoad(sampleCls, lib);
+
+                if (ok)
+                    loadedLibs.add(lib);
             }
+
+            loadedLibNames = loadedLibs.toArray(new String[loadedLibs.size()]);
         }
         catch (Exception e) {
-            U.quietAndWarn(null, "Failed to initialize Hadoop native library " +
-                "(native Hadoop methods might not work properly): " + e);
+            throw new IgniteException(e);
+        }
+
+    }
+
+    /**
+     *
+     * @param ldr
+     * @param target
+     */
+    private void collectNativeLibrariesFromLoader(ClassLoader ldr, Collection<Object>
target) {
+        final Vector vector = U.field(ldr, "nativeLibraries");
+
+        for (Object lib : vector) {
+            String name = U.field(lib, "name");
+
+            // TODO: LIBHADOOP should be added implicitly into "libNames"
+            boolean addLib = false; //name.contains(LIBHADOOP);
+
+            if (loadedLibNames != null) {
+                for (String libName : loadedLibNames) {
+                    if (libName != null && name.contains(libName)) {
+                        addLib = true;
+
+                        break;
+                    }
+                }
+            }
+
+            if (addLib)
+                target.add(lib);
         }
     }
 
+
     /** {@inheritDoc} */
     @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException
{
         try {

http://git-wip-us.apache.org/repos/asf/ignite/blob/df3ad76f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
index 3415d6a..aeef031 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderUtils.java
@@ -90,7 +90,7 @@ public class HadoopClassLoaderUtils {
      * @return {@code true} If this is Hadoop class.
      */
     public static boolean isHadoop(String cls) {
-        return cls.startsWith("org.apache.hadoop.");
+        return cls.startsWith("org.apache.hadoop.") || cls.equals(XXX.class.getName());
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/df3ad76f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
new file mode 100644
index 0000000..a795085
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/LoadHelper.java
@@ -0,0 +1,42 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.lang.reflect.Method;
+
+/**
+ * Should be loaded with the
+ */
+public class LoadHelper {
+
+    private static Method method;
+
+    static {
+        try {
+            method = ClassLoader.class.getDeclaredMethod("loadLibrary",
+                new Class[] {Class.class, String.class, boolean.class});
+
+            method.setAccessible(true);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * Utility method that loads given class by name with the given "caller" class.
+     *
+     * @return 'true' on success.
+     */
+    public static boolean tryLoad(Class caller, String libName) {
+        try {
+            method.invoke(null, new Object[] {caller/*caller class*/, libName /*lib*/ , false/*isAbsolute*/
});
+
+            return true;
+        }
+        catch (Throwable t) {
+            // TODO:
+            t.printStackTrace();
+
+            return false;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/df3ad76f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/XXX.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/XXX.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/XXX.java
new file mode 100644
index 0000000..b230636
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/XXX.java
@@ -0,0 +1,7 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Created by ivan on 19.09.16.
+ */
+public class XXX {
+}


Mime
View raw message