ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject ignite git commit: IGNITE-2218: fix for native libs loading
Date Wed, 23 Dec 2015 18:51:39 GMT
Repository: ignite
Updated Branches:
  refs/heads/ignite-igfs-fixes bd05fa90b -> b8bcb3f8c


IGNITE-2218: fix for native libs loading


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b8bcb3f8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b8bcb3f8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b8bcb3f8

Branch: refs/heads/ignite-igfs-fixes
Commit: b8bcb3f8cba15f778218f9defb623ad7436de385
Parents: bd05fa9
Author: dmagda <magda7817@gmail.com>
Authored: Wed Dec 23 21:51:30 2015 +0300
Committer: dmagda <magda7817@gmail.com>
Committed: Wed Dec 23 21:51:30 2015 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopClassLoader.java    |  6 ++
 .../v2/HadoopJniBasedUnixGroupsMapping.java     | 98 ++++++++++++++++++++
 .../hadoop/v2/HadoopNativeCodeLoader.java       | 57 ++++++++++--
 3 files changed, 155 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/b8bcb3f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index f12af46..cf11b38 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -33,6 +33,7 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon;
+import org.apache.ignite.internal.processors.hadoop.v2.HadoopJniBasedUnixGroupsMapping;
 import org.apache.ignite.internal.processors.hadoop.v2.HadoopNativeCodeLoader;
 import org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager;
 import org.apache.ignite.internal.util.typedef.F;
@@ -67,6 +68,9 @@ public class HadoopClassLoader extends URLClassLoader {
     public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon";
 
     /** */
+    private static final String HADOOP_UNIX_MAPPING_CLASS_NAME = "org.apache.hadoop.security.JniBasedUnixGroupsMapping";
+
+    /** */
     private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)HadoopClassLoader.class.getClassLoader();
 
     /** */
@@ -147,6 +151,8 @@ public class HadoopClassLoader extends URLClassLoader {
                     return loadFromBytes(name, HadoopShutdownHookManager.class.getName());
                 else if (name.endsWith(".util.NativeCodeLoader"))
                     return loadFromBytes(name, HadoopNativeCodeLoader.class.getName());
+                else if (name.equals(HADOOP_UNIX_MAPPING_CLASS_NAME))
+                    return loadFromBytes(name, HadoopJniBasedUnixGroupsMapping.class.getName());
                 else if (name.equals(HADOOP_DAEMON_CLASS_NAME))
                     // We replace this in order to be able to forcibly stop some daemon threads
                     // that otherwise never stop (e.g. PeerCache runnables):

http://git-wip-us.apache.org/repos/asf/ignite/blob/b8bcb3f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
new file mode 100644
index 0000000..d917e2b
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopJniBasedUnixGroupsMapping.java
@@ -0,0 +1,98 @@
+package org.apache.ignite.internal.processors.hadoop.v2;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.apache.hadoop.security.JniBasedUnixGroupsMapping;
+
+/**
+ * A JNI-based implementation of {@link GroupMappingServiceProvider}
+ * that invokes libC calls to get the group
+ * memberships of a given user.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class HadoopJniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
+
+    private static final Log LOG =
+        LogFactory.getLog(JniBasedUnixGroupsMapping.class);
+
+    static {
+        if (!HadoopNativeCodeLoader.isNativeCodeLoaded()) {
+            throw new RuntimeException("Bailing out since native library couldn't " +
+                "be loaded");
+        }
+        anchorNative();
+        LOG.debug("Using JniBasedUnixGroupsMapping for Group resolution");
+    }
+
+    /**
+     * Set up our JNI resources.
+     *
+     * @throws                 RuntimeException if setup fails.
+     */
+    static void anchorNative() {
+        try {
+            ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+            Class cls = Class.forName(JniBasedUnixGroupsMapping.class.getName(), true, parent);
+
+            Method m = cls.getDeclaredMethod("anchorNative");
+
+            m.setAccessible(true);
+
+            m.invoke(null);
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Get the set of groups associated with a user.
+     *
+     * @param username           The user name
+     *
+     * @return                   The set of groups associated with a user.
+     */
+    native static String[] getGroupsForUser(String username);
+
+    /**
+     * Log an error message about a group.  Used from JNI.
+     */
+    static private void logError(int groupId, String error) {
+        LOG.error("error looking up the name of group " + groupId + ": " + error);
+    }
+
+    @Override
+    public List<String> getGroups(String user) throws IOException {
+        String[] groups = new String[0];
+        try {
+            groups = getGroupsForUser(user);
+        } catch (Exception e) {
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Error getting groups for " + user, e);
+            } else {
+                LOG.info("Error getting groups for " + user + ": " + e.getMessage());
+            }
+        }
+        return Arrays.asList(groups);
+    }
+
+    @Override
+    public void cacheGroupsRefresh() throws IOException {
+        // does nothing in this provider of user to groups mapping
+    }
+
+    @Override
+    public void cacheGroupsAdd(List<String> groups) throws IOException {
+        // does nothing in this provider of user to groups mapping
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b8bcb3f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
index 4c4840d..f3ed66c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopNativeCodeLoader.java
@@ -17,9 +17,12 @@
 
 package org.apache.ignite.internal.processors.hadoop.v2;
 
+import java.lang.reflect.Method;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.util.NativeCodeLoader;
 
 /**
  * A fake helper to load the native hadoop code i.e. libhadoop.so.
@@ -34,23 +37,62 @@ public class HadoopNativeCodeLoader {
      *         else <code>false</code>
      */
     public static boolean isNativeCodeLoaded() {
-        return false;
+        try {
+            ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+            Class cls = Class.forName(NativeCodeLoader.class.getName(), true, parent);
+
+            Method m = cls.getMethod("isNativeCodeLoaded");
+
+            boolean value = (boolean)m.invoke(null);
+
+            return value;
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
     }
 
     /**
      * Returns true only if this build was compiled with support for snappy.
      */
     public static boolean buildSupportsSnappy() {
-        return false;
+        try {
+            ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+            Class cls = Class.forName(NativeCodeLoader.class.getName(), true, parent);
+
+            boolean value = (boolean)cls.getMethod("buildSupportsSnappy").invoke(null);
+
+            return value;
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
     }
 
     /**
      * @return Library name.
      */
     public static String getLibraryName() {
-        throw new IllegalStateException();
+        try {
+            ClassLoader parent = Configuration.class.getClassLoader().getParent();
+
+            Class cls = Class.forName(NativeCodeLoader.class.getName(), true, parent);
+
+            Method m = cls.getMethod("getLibraryName");
+
+            String value = (String)m.invoke(null);
+
+            return value;
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
     }
 
+    // --------------- copied methods:
+
     /**
      * Return if native hadoop libraries, if present, can be used for this job.
      * @param conf configuration
@@ -59,7 +101,8 @@ public class HadoopNativeCodeLoader {
      *         used for this job; <code>false</code> otherwise.
      */
     public boolean getLoadNativeLibraries(Configuration conf) {
-        return false;
+        return conf.getBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
+            CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
     }
 
     /**
@@ -68,7 +111,9 @@ public class HadoopNativeCodeLoader {
      * @param conf configuration
      * @param loadNativeLibraries can native hadoop libraries be loaded
      */
-    public void setLoadNativeLibraries(Configuration conf, boolean loadNativeLibraries) {
-        // No-op.
+    public void setLoadNativeLibraries(Configuration conf,
+                                       boolean loadNativeLibraries) {
+        conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
+            loadNativeLibraries);
     }
 }


Mime
View raw message