ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [49/50] ignite git commit: FIxed.
Date Mon, 20 Jun 2016 12:53:29 GMT
FIxed.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/699ff6a1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/699ff6a1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/699ff6a1

Branch: refs/heads/ignite-3341
Commit: 699ff6a16d6b0edc46001d06cd9e0e45d7970ac3
Parents: c952def
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Mon Jun 20 12:22:38 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Mon Jun 20 12:22:38 2016 +0300

----------------------------------------------------------------------
 .../configuration/HadoopConfiguration.java      | 38 ++++++++++++++++++++
 .../processors/hadoop/HadoopJobInfo.java        |  5 ++-
 .../processors/hadoop/HadoopClassLoader.java    | 14 ++++++--
 .../processors/hadoop/HadoopDefaultJobInfo.java |  9 ++---
 .../hadoop/jobtracker/HadoopJobTracker.java     |  8 +++--
 .../processors/hadoop/v2/HadoopV2Job.java       | 14 ++++++--
 .../hadoop/HadoopClassLoaderTest.java           |  2 +-
 .../processors/hadoop/HadoopSnappyTest.java     |  2 +-
 .../collections/HadoopAbstractMapTest.java      |  7 ++--
 9 files changed, 83 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
b/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
index 51aa291..7b87ecb 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java
@@ -17,8 +17,13 @@
 
 package org.apache.ignite.configuration;
 
+import org.apache.ignite.lifecycle.LifecycleBean;
 import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlanner;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+import java.util.List;
 
 /**
  * Ignite Hadoop Accelerator configuration.
@@ -51,6 +56,9 @@ public class HadoopConfiguration {
     /** */
     private int maxTaskQueueSize = DFLT_MAX_TASK_QUEUE_SIZE;
 
+    /** Library names. */
+    private Collection<String> libNames;
+
     /**
      * Default constructor.
      */
@@ -71,6 +79,7 @@ public class HadoopConfiguration {
         planner = cfg.getMapReducePlanner();
         maxParallelTasks = cfg.getMaxParallelTasks();
         maxTaskQueueSize = cfg.getMaxTaskQueueSize();
+        libNames = cfg.getNativeLibraryNames();
     }
 
     /**
@@ -169,6 +178,35 @@ public class HadoopConfiguration {
         this.planner = planner;
     }
 
+    /**
+     * Get native library names.
+     * <p>
+     * Ignite Hadoop Accelerator executes all Hadoop jobs and tasks in the same process,
isolating them with help
+     * of classloaders. If Hadoop job or task loads a native library, it might lead to exception,
because Java do
+     * not allow to load the same library multiple times from different classloaders. To
overcome the problem,
+     * you should to the following:
+     * <ul>
+     *     <li>Load necessary libraries in advance from base classloader; {@link LifecycleBean}
is a good candidate
+     *     for this;</li>
+     *     <li>Add names of loaded libraries to this property, so that we can link
them.</li>
+     *     <li>Remove {@link System#load(String)} and {@link System#loadLibrary(String)}
calls from your job/task.</li>     *
+     * </ul>
+     *
+     * @return Native library names.
+     */
+    @Nullable public Collection<String> getNativeLibraryNames() {
+        return libNames;
+    }
+
+    /**
+     * Set native library names. See {@link #getNativeLibraryNames()} for more information.
+     *
+     * @param libNames Native library names.
+     */
+    public void setNativeLibraryNames(@Nullable Collection<String> libNames) {
+        this.libNames = libNames;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(HadoopConfiguration.class, this, super.toString());

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
index eda8e97..832cb20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.processors.hadoop;
 
 import java.io.Serializable;
+import java.util.Collection;
+
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.jetbrains.annotations.Nullable;
@@ -58,11 +60,12 @@ public interface HadoopJobInfo extends Serializable {
      * @param jobCls The job class.
      * @param jobId Job ID.
      * @param log Logger.
+     * @param libNames Optional additional native library names.
      * @return Job.
      * @throws IgniteCheckedException If failed.
      */
     public HadoopJob createJob(Class<? extends HadoopJob> jobCls,
-        HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException;
+        HadoopJobId jobId, IgniteLogger log, @Nullable Collection<String> libNames)
throws IgniteCheckedException;
 
     /**
      * @return Number of reducers configured for job.

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 4448b2d..fdbf06f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -99,6 +99,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
     @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
     private final String name;
 
+    /** Native library names. */
+    private final Collection<String> libNames;
+
     /**
      * Gets name for Job class loader. The name is specific for local node id.
      * @param locNodeId The local node id.
@@ -122,14 +125,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
     }
 
     /**
+     * Constructor.
+     *
      * @param urls Urls.
+     * @param name Classloader name.
+     * @param libNames Optional additional native library names to be linked from parent
classloader.
      */
-    public HadoopClassLoader(URL[] urls, String name) {
+    public HadoopClassLoader(URL[] urls, String name, @Nullable Collection<String>
libNames) {
         super(addHadoopUrls(urls), APP_CLS_LDR);
 
         assert !(getParent() instanceof HadoopClassLoader);
 
         this.name = name;
+        this.libNames = libNames;
 
         initializeNativeLibraries();
     }
@@ -161,7 +169,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
                 for (Object lib : vector) {
                     String libName = U.field(lib, "name");
 
-                    if (libName.contains(LIBHADOOP)) {
+                    boolean add = libName.contains(LIBHADOOP) || (libNames != null &&
libNames.contains(libName));
+
+                    if (add) {
                         curVector.add(lib);
 
                         return;

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
index fe125fe..6ca6f1a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.lang.reflect.Constructor;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.IgniteCheckedException;
@@ -82,15 +83,15 @@ public class HadoopDefaultJobInfo implements HadoopJobInfo, Externalizable
{
     }
 
     /** {@inheritDoc} */
-    @Override public HadoopJob createJob(Class<? extends HadoopJob> jobCls,
-            HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
+    @Override public HadoopJob createJob(Class<? extends HadoopJob> jobCls, HadoopJobId
jobId, IgniteLogger log,
+        @Nullable Collection<String> libNames) throws IgniteCheckedException {
         assert jobCls != null;
 
         try {
             Constructor<? extends HadoopJob> constructor = jobCls.getConstructor(HadoopJobId.class,
-                HadoopDefaultJobInfo.class, IgniteLogger.class);
+                HadoopDefaultJobInfo.class, IgniteLogger.class, Collection.class);
 
-            return constructor.newInstance(jobId, this, log);
+            return constructor.newInstance(jobId, this, log, libNames);
         }
         // NB: java.lang.NoClassDefFoundError may be thrown from Class#getConstructor() call.
         catch (Throwable t) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index f4cf892..f3294bf 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -157,7 +157,8 @@ public class HadoopJobTracker extends HadoopComponent {
 
         assert jobCls == null;
 
-        HadoopClassLoader ldr = new HadoopClassLoader(null, HadoopClassLoader.nameForJob(nodeId));
+        HadoopClassLoader ldr = new HadoopClassLoader(null, HadoopClassLoader.nameForJob(nodeId),
+            ctx.configuration().getNativeLibraryNames());
 
         try {
             jobCls = (Class<HadoopV2Job>)ldr.loadClass(HadoopV2Job.class.getName());
@@ -727,6 +728,7 @@ public class HadoopJobTracker extends HadoopComponent {
      * @param jobId  Job ID.
      * @param plan Map-reduce plan.
      */
+    @SuppressWarnings({"unused", "ConstantConditions" })
     private void printPlan(HadoopJobId jobId, HadoopMapReducePlan plan) {
         log.info("Plan for " + jobId);
 
@@ -886,6 +888,8 @@ public class HadoopJobTracker extends HadoopComponent {
                     finishFut.onDone(jobId, meta.failCause());
                 }
 
+                assert job != null;
+
                 if (ctx.jobUpdateLeader())
                     job.cleanupStagingDirectory();
 
@@ -1052,7 +1056,7 @@ public class HadoopJobTracker extends HadoopComponent {
                 jobInfo = meta.jobInfo();
             }
 
-            job = jobInfo.createJob(jobCls, jobId, log);
+            job = jobInfo.createJob(jobCls, jobId, log, ctx.configuration().getNativeLibraryNames());
 
             job.initialize(false, ctx.localNodeId());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index b69447d..edd5269 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -93,6 +93,9 @@ public class HadoopV2Job implements HadoopJob {
     /** Job info. */
     protected final HadoopJobInfo jobInfo;
 
+    /** Native library names. */
+    private final Collection<String> libNames;
+
     /** */
     private final JobID hadoopJobID;
 
@@ -119,16 +122,21 @@ public class HadoopV2Job implements HadoopJob {
     private volatile byte[] jobConfData;
 
     /**
+     * Constructor.
+     *
      * @param jobId Job ID.
      * @param jobInfo Job info.
      * @param log Logger.
+     * @param libNames Optional additional native library names.
      */
-    public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, IgniteLogger
log) {
+    public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, IgniteLogger
log,
+        @Nullable Collection<String> libNames) {
         assert jobId != null;
         assert jobInfo != null;
 
         this.jobId = jobId;
         this.jobInfo = jobInfo;
+        this.libNames = libNames;
 
         hadoopJobID = new JobID(jobId.globalId().toString(), jobId.localId());
 
@@ -220,7 +228,7 @@ public class HadoopV2Job implements HadoopJob {
     }
 
     /** {@inheritDoc} */
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings({"unchecked", "MismatchedQueryAndUpdateOfCollection" })
     @Override public HadoopTaskContext getTaskContext(HadoopTaskInfo info) throws IgniteCheckedException
{
         T2<HadoopTaskType, Integer> locTaskId = new T2<>(info.type(),  info.taskNumber());
 
@@ -242,7 +250,7 @@ public class HadoopV2Job implements HadoopJob {
                 // Note that the classloader identified by the task it was initially created
for,
                 // but later it may be reused for other tasks.
                 HadoopClassLoader ldr = new HadoopClassLoader(rsrcMgr.classPath(),
-                    HadoopClassLoader.nameForTask(info, false));
+                    HadoopClassLoader.nameForTask(info, false), libNames);
 
                 cls = (Class<? extends HadoopTaskContext>)ldr.loadClass(HadoopV2TaskContext.class.getName());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
index 55fac2c..02d98d0 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java
@@ -49,7 +49,7 @@ import org.apache.ignite.internal.processors.hadoop.deps.Without;
  */
 public class HadoopClassLoaderTest extends TestCase {
     /** */
-    final HadoopClassLoader ldr = new HadoopClassLoader(null, "test");
+    final HadoopClassLoader ldr = new HadoopClassLoader(null, "test", null);
 
     /**
      * @throws Exception If failed.

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
index 014ff1e..b4e3dc2 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java
@@ -49,7 +49,7 @@ public class HadoopSnappyTest extends GridCommonAbstractTest {
 
         // Run the same in several more class loaders simulating jobs and tasks:
         for (int i = 0; i < 2; i++) {
-            ClassLoader hadoopClsLdr = new HadoopClassLoader(null, "cl-" + i);
+            ClassLoader hadoopClsLdr = new HadoopClassLoader(null, "cl-" + i, null);
 
             Class<?> cls = (Class)Class.forName(HadoopSnappyTest.class.getName(), true,
hadoopClsLdr);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
index 493098f..a9d2116 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
 
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.concurrent.Callable;
 import org.apache.commons.collections.comparators.ComparableComparator;
@@ -77,11 +78,13 @@ public abstract class HadoopAbstractMapTest extends GridCommonAbstractTest
{
         }
 
         /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
         @Override public Comparator<Object> sortComparator() {
             return ComparableComparator.getInstance();
         }
 
         /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
         @Override public Comparator<Object> groupComparator() {
             return ComparableComparator.getInstance();
         }
@@ -141,8 +144,8 @@ public abstract class HadoopAbstractMapTest extends GridCommonAbstractTest
{
         }
 
         /** {@inheritDoc} */
-        @Override public HadoopJob createJob(Class<? extends HadoopJob> jobCls,
-                HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
+        @Override public HadoopJob createJob(Class<? extends HadoopJob> jobCls, HadoopJobId
jobId, IgniteLogger log,
+            @Nullable Collection<String> libNames) throws IgniteCheckedException {
             assert false;
 
             return null;


Mime
View raw message