Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8815F200B50 for ; Mon, 20 Jun 2016 14:52:47 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 86D58160A55; Mon, 20 Jun 2016 12:52:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2CD7E160A24 for ; Mon, 20 Jun 2016 14:52:44 +0200 (CEST) Received: (qmail 9077 invoked by uid 500); 20 Jun 2016 12:52:43 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 7942 invoked by uid 99); 20 Jun 2016 12:52:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 20 Jun 2016 12:52:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 88108DFE65; Mon, 20 Jun 2016 12:52:42 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vozerov@apache.org To: commits@ignite.apache.org Date: Mon, 20 Jun 2016 12:53:29 -0000 Message-Id: <0d7ce9f55ba74f0db44ea18a3d5080d6@git.apache.org> In-Reply-To: <89985bfe12394840b4deddba739acaf2@git.apache.org> References: <89985bfe12394840b4deddba739acaf2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [49/50] ignite git commit: FIxed. archived-at: Mon, 20 Jun 2016 12:52:47 -0000 FIxed. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/699ff6a1 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/699ff6a1 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/699ff6a1 Branch: refs/heads/ignite-3341 Commit: 699ff6a16d6b0edc46001d06cd9e0e45d7970ac3 Parents: c952def Author: vozerov-gridgain Authored: Mon Jun 20 12:22:38 2016 +0300 Committer: vozerov-gridgain Committed: Mon Jun 20 12:22:38 2016 +0300 ---------------------------------------------------------------------- .../configuration/HadoopConfiguration.java | 38 ++++++++++++++++++++ .../processors/hadoop/HadoopJobInfo.java | 5 ++- .../processors/hadoop/HadoopClassLoader.java | 14 ++++++-- .../processors/hadoop/HadoopDefaultJobInfo.java | 9 ++--- .../hadoop/jobtracker/HadoopJobTracker.java | 8 +++-- .../processors/hadoop/v2/HadoopV2Job.java | 14 ++++++-- .../hadoop/HadoopClassLoaderTest.java | 2 +- .../processors/hadoop/HadoopSnappyTest.java | 2 +- .../collections/HadoopAbstractMapTest.java | 7 ++-- 9 files changed, 83 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java index 51aa291..7b87ecb 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/HadoopConfiguration.java @@ -17,8 +17,13 @@ package org.apache.ignite.configuration; +import org.apache.ignite.lifecycle.LifecycleBean; import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlanner; import org.apache.ignite.internal.util.typedef.internal.S; +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; +import java.util.List; /** * Ignite Hadoop Accelerator configuration. @@ -51,6 +56,9 @@ public class HadoopConfiguration { /** */ private int maxTaskQueueSize = DFLT_MAX_TASK_QUEUE_SIZE; + /** Library names. */ + private Collection libNames; + /** * Default constructor. */ @@ -71,6 +79,7 @@ public class HadoopConfiguration { planner = cfg.getMapReducePlanner(); maxParallelTasks = cfg.getMaxParallelTasks(); maxTaskQueueSize = cfg.getMaxTaskQueueSize(); + libNames = cfg.getNativeLibraryNames(); } /** @@ -169,6 +178,35 @@ public class HadoopConfiguration { this.planner = planner; } + /** + * Get native library names. + *

+ * Ignite Hadoop Accelerator executes all Hadoop jobs and tasks in the same process, isolating them with help + * of classloaders. If Hadoop job or task loads a native library, it might lead to exception, because Java do + * not allow to load the same library multiple times from different classloaders. To overcome the problem, + * you should to the following: + *

    + *
  • Load necessary libraries in advance from base classloader; {@link LifecycleBean} is a good candidate + * for this;
    • + *
  • Add names of loaded libraries to this property, so that we can link them.
    • + *
  • Remove {@link System#load(String)} and {@link System#loadLibrary(String)} calls from your job/task.
    • * + *
+ * + * @return Native library names. + */ + @Nullable public Collection getNativeLibraryNames() { + return libNames; + } + + /** + * Set native library names. See {@link #getNativeLibraryNames()} for more information. + * + * @param libNames Native library names. + */ + public void setNativeLibraryNames(@Nullable Collection libNames) { + this.libNames = libNames; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(HadoopConfiguration.class, this, super.toString()); http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java index eda8e97..832cb20 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java @@ -18,6 +18,8 @@ package org.apache.ignite.internal.processors.hadoop; import java.io.Serializable; +import java.util.Collection; + import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.jetbrains.annotations.Nullable; @@ -58,11 +60,12 @@ public interface HadoopJobInfo extends Serializable { * @param jobCls The job class. * @param jobId Job ID. * @param log Logger. + * @param libNames Optional additional native library names. * @return Job. * @throws IgniteCheckedException If failed. */ public HadoopJob createJob(Class jobCls, - HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException; + HadoopJobId jobId, IgniteLogger log, @Nullable Collection libNames) throws IgniteCheckedException; /** * @return Number of reducers configured for job. http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java index 4448b2d..fdbf06f 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java @@ -99,6 +99,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) private final String name; + /** Native library names. */ + private final Collection libNames; + /** * Gets name for Job class loader. The name is specific for local node id. * @param locNodeId The local node id. @@ -122,14 +125,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { } /** + * Constructor. + * * @param urls Urls. + * @param name Classloader name. + * @param libNames Optional additional native library names to be linked from parent classloader. */ - public HadoopClassLoader(URL[] urls, String name) { + public HadoopClassLoader(URL[] urls, String name, @Nullable Collection libNames) { super(addHadoopUrls(urls), APP_CLS_LDR); assert !(getParent() instanceof HadoopClassLoader); this.name = name; + this.libNames = libNames; initializeNativeLibraries(); } @@ -161,7 +169,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache { for (Object lib : vector) { String libName = U.field(lib, "name"); - if (libName.contains(LIBHADOOP)) { + boolean add = libName.contains(LIBHADOOP) || (libNames != null && libNames.contains(libName)); + + if (add) { curVector.add(lib); return; http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java index fe125fe..6ca6f1a 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.lang.reflect.Constructor; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import org.apache.ignite.IgniteCheckedException; @@ -82,15 +83,15 @@ public class HadoopDefaultJobInfo implements HadoopJobInfo, Externalizable { } /** {@inheritDoc} */ - @Override public HadoopJob createJob(Class jobCls, - HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException { + @Override public HadoopJob createJob(Class jobCls, HadoopJobId jobId, IgniteLogger log, + @Nullable Collection libNames) throws IgniteCheckedException { assert jobCls != null; try { Constructor constructor = jobCls.getConstructor(HadoopJobId.class, - HadoopDefaultJobInfo.class, IgniteLogger.class); + HadoopDefaultJobInfo.class, IgniteLogger.class, Collection.class); - return constructor.newInstance(jobId, this, log); + return constructor.newInstance(jobId, this, log, libNames); } // NB: java.lang.NoClassDefFoundError may be thrown from Class#getConstructor() call. catch (Throwable t) { http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java index f4cf892..f3294bf 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java @@ -157,7 +157,8 @@ public class HadoopJobTracker extends HadoopComponent { assert jobCls == null; - HadoopClassLoader ldr = new HadoopClassLoader(null, HadoopClassLoader.nameForJob(nodeId)); + HadoopClassLoader ldr = new HadoopClassLoader(null, HadoopClassLoader.nameForJob(nodeId), + ctx.configuration().getNativeLibraryNames()); try { jobCls = (Class)ldr.loadClass(HadoopV2Job.class.getName()); @@ -727,6 +728,7 @@ public class HadoopJobTracker extends HadoopComponent { * @param jobId Job ID. * @param plan Map-reduce plan. */ + @SuppressWarnings({"unused", "ConstantConditions" }) private void printPlan(HadoopJobId jobId, HadoopMapReducePlan plan) { log.info("Plan for " + jobId); @@ -886,6 +888,8 @@ public class HadoopJobTracker extends HadoopComponent { finishFut.onDone(jobId, meta.failCause()); } + assert job != null; + if (ctx.jobUpdateLeader()) job.cleanupStagingDirectory(); @@ -1052,7 +1056,7 @@ public class HadoopJobTracker extends HadoopComponent { jobInfo = meta.jobInfo(); } - job = jobInfo.createJob(jobCls, jobId, log); + job = jobInfo.createJob(jobCls, jobId, log, ctx.configuration().getNativeLibraryNames()); job.initialize(false, ctx.localNodeId()); http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java index b69447d..edd5269 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java @@ -93,6 +93,9 @@ public class HadoopV2Job implements HadoopJob { /** Job info. */ protected final HadoopJobInfo jobInfo; + /** Native library names. */ + private final Collection libNames; + /** */ private final JobID hadoopJobID; @@ -119,16 +122,21 @@ public class HadoopV2Job implements HadoopJob { private volatile byte[] jobConfData; /** + * Constructor. + * * @param jobId Job ID. * @param jobInfo Job info. * @param log Logger. + * @param libNames Optional additional native library names. */ - public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, IgniteLogger log) { + public HadoopV2Job(HadoopJobId jobId, final HadoopDefaultJobInfo jobInfo, IgniteLogger log, + @Nullable Collection libNames) { assert jobId != null; assert jobInfo != null; this.jobId = jobId; this.jobInfo = jobInfo; + this.libNames = libNames; hadoopJobID = new JobID(jobId.globalId().toString(), jobId.localId()); @@ -220,7 +228,7 @@ public class HadoopV2Job implements HadoopJob { } /** {@inheritDoc} */ - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked", "MismatchedQueryAndUpdateOfCollection" }) @Override public HadoopTaskContext getTaskContext(HadoopTaskInfo info) throws IgniteCheckedException { T2 locTaskId = new T2<>(info.type(), info.taskNumber()); @@ -242,7 +250,7 @@ public class HadoopV2Job implements HadoopJob { // Note that the classloader identified by the task it was initially created for, // but later it may be reused for other tasks. HadoopClassLoader ldr = new HadoopClassLoader(rsrcMgr.classPath(), - HadoopClassLoader.nameForTask(info, false)); + HadoopClassLoader.nameForTask(info, false), libNames); cls = (Class)ldr.loadClass(HadoopV2TaskContext.class.getName()); http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java index 55fac2c..02d98d0 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java @@ -49,7 +49,7 @@ import org.apache.ignite.internal.processors.hadoop.deps.Without; */ public class HadoopClassLoaderTest extends TestCase { /** */ - final HadoopClassLoader ldr = new HadoopClassLoader(null, "test"); + final HadoopClassLoader ldr = new HadoopClassLoader(null, "test", null); /** * @throws Exception If failed. http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java index 014ff1e..b4e3dc2 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopSnappyTest.java @@ -49,7 +49,7 @@ public class HadoopSnappyTest extends GridCommonAbstractTest { // Run the same in several more class loaders simulating jobs and tasks: for (int i = 0; i < 2; i++) { - ClassLoader hadoopClsLdr = new HadoopClassLoader(null, "cl-" + i); + ClassLoader hadoopClsLdr = new HadoopClassLoader(null, "cl-" + i, null); Class cls = (Class)Class.forName(HadoopSnappyTest.class.getName(), true, hadoopClsLdr); http://git-wip-us.apache.org/repos/asf/ignite/blob/699ff6a1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java index 493098f..a9d2116 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopAbstractMapTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.hadoop.shuffle.collections; +import java.util.Collection; import java.util.Comparator; import java.util.concurrent.Callable; import org.apache.commons.collections.comparators.ComparableComparator; @@ -77,11 +78,13 @@ public abstract class HadoopAbstractMapTest extends GridCommonAbstractTest { } /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override public Comparator sortComparator() { return ComparableComparator.getInstance(); } /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override public Comparator groupComparator() { return ComparableComparator.getInstance(); } @@ -141,8 +144,8 @@ public abstract class HadoopAbstractMapTest extends GridCommonAbstractTest { } /** {@inheritDoc} */ - @Override public HadoopJob createJob(Class jobCls, - HadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException { + @Override public HadoopJob createJob(Class jobCls, HadoopJobId jobId, IgniteLogger log, + @Nullable Collection libNames) throws IgniteCheckedException { assert false; return null;