ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [06/25] ignite git commit: IGNITE-3383: Hadoop: Improved Hadoop component start logic.
Date Fri, 01 Jul 2016 12:59:54 GMT
IGNITE-3383: Hadoop: Improved Hadoop component start logic.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bdeef4c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bdeef4c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bdeef4c0

Branch: refs/heads/ignite-3154
Commit: bdeef4c06a807ca9a4e7f2ed4faa4ca3f0d26f20
Parents: a77e690
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Tue Jun 28 16:33:09 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Tue Jun 28 16:33:09 2016 +0300

----------------------------------------------------------------------
 .../apache/ignite/internal/IgniteKernal.java    | 48 ++++++++++++------
 .../processors/hadoop/HadoopNoopProcessor.java  |  5 ++
 .../hadoop/HadoopProcessorAdapter.java          |  7 +++
 .../processors/hadoop/HadoopClassLoader.java    |  2 +-
 .../processors/hadoop/HadoopProcessor.java      | 52 +++++++++-----------
 5 files changed, 68 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 3572293..b6ba4cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -114,7 +114,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor;
+import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
 import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor;
@@ -1087,26 +1087,44 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable
{
      * Create Hadoop component.
      *
      * @return Non-null Hadoop component: workable or no-op.
-     * @throws IgniteCheckedException if the component is mandatory and cannot be initialized.
+     * @throws IgniteCheckedException If the component is mandatory and cannot be initialized.
      */
-    private GridComponent createHadoopComponent() throws IgniteCheckedException {
-        GridComponent cmp;
+    private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedException
{
+        boolean mandatory = cfg.getHadoopConfiguration() != null;
 
-        if (cfg.isPeerClassLoadingEnabled()) {
-            cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+        if (mandatory) {
+            if (cfg.isPeerClassLoadingEnabled())
+                throw new IgniteCheckedException("Hadoop module cannot be used with peer
class loading enabled " +
+                    "(set IgniteConfiguration.peerClassLoadingEnabled to \"false\").");
 
-            if (!(cmp instanceof HadoopNoopProcessor)) {
-                U.warn(log, "Hadoop module is found in classpath, but it will not be started
because peer class " +
-                    "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled
to \"false\" to start " +
-                    "Hadoop module).");
+            return IgniteComponentType.HADOOP.createIfInClassPath(ctx, true);
+        }
+        else {
+            HadoopProcessorAdapter cmp = null;
 
-                cmp = IgniteComponentType.HADOOP.create(ctx, true/*no-op*/);
+            if (IgniteComponentType.HADOOP.inClassPath() && cfg.isPeerClassLoadingEnabled())
{
+                U.warn(log, "Hadoop module is found in classpath, but will not be started
because peer class " +
+                    "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled
to \"false\" if you want " +
+                    "to use Hadoop module).");
             }
-        }
-        else
-            cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration()
!= null);
+            else {
+                cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+
+                try {
+                    cmp.validateEnvironment();
+                }
+                catch (IgniteException | IgniteCheckedException e) {
+                    U.quietAndWarn(log, "Hadoop module will not start due to exception: "
+ e.getMessage());
+
+                    cmp = null;
+                }
+            }
+
+            if (cmp == null)
+                cmp = IgniteComponentType.HADOOP.create(ctx, true);
 
-        return cmp;
+            return cmp;
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
index 72b5050..065cd72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
@@ -74,4 +74,9 @@ public class HadoopNoopProcessor extends HadoopProcessorAdapter {
     @Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
         return false;
     }
+
+    /** {@inheritDoc} */
+    @Override public void validateEnvironment() throws IgniteCheckedException {
+        // No-oop.
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
index 3d381c3..f914639 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
@@ -94,4 +94,11 @@ public abstract class HadoopProcessorAdapter extends GridProcessorAdapter
{
      * @throws IgniteCheckedException If failed.
      */
     public abstract boolean kill(HadoopJobId jobId) throws IgniteCheckedException;
+
+    /**
+     * Validate Hadoop environment.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public abstract void validateEnvironment() throws IgniteCheckedException;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 1c844c4..389de8c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -501,7 +501,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
                 hadoopUrls = HadoopClasspathUtils.classpathUrls();
             }
             catch (IOException e) {
-                throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.",
e);
+                throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations:
" + e.getMessage(), e);
             }
 
             hadoopJars = hadoopUrls;

http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index c45953e..7fc7499 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -50,6 +50,8 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
     private Hadoop hadoop;
 
     /**
+     * Constructor.
+     *
      * @param ctx Kernal context.
      */
     public HadoopProcessor(GridKernalContext ctx) {
@@ -70,24 +72,6 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
 
         initializeDefaults(cfg);
 
-        validate(cfg);
-
-        try {
-            HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
-
-            if (loc.home() != null)
-                U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
-
-            U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
-            U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
-            U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
-        }
-        catch (IOException ioe) {
-            throw new IgniteCheckedException(ioe);
-        }
-
-        HadoopClassLoader.hadoopUrls();
-
         hctx = new HadoopContext(
             ctx,
             cfg,
@@ -204,6 +188,26 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
         return hctx.jobTracker().killJob(jobId);
     }
 
+    /** {@inheritDoc} */
+    @Override public void validateEnvironment() throws IgniteCheckedException {
+        // Perform some static checks as early as possible, so that any recoverable exceptions
are thrown here.
+        try {
+            HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
+
+            if (loc.home() != null)
+                U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
+
+            U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
+            U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
+            U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
+        }
+        catch (IOException ioe) {
+            throw new IgniteCheckedException(ioe.getMessage(), ioe);
+        }
+
+        HadoopClassLoader.hadoopUrls();
+    }
+
     /**
      * Initializes default hadoop configuration.
      *
@@ -213,16 +217,4 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
         if (cfg.getMapReducePlanner() == null)
             cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner());
     }
-
-    /**
-     * Validates Grid and Hadoop configuration for correctness.
-     *
-     * @param hadoopCfg Hadoop configuration.
-     * @throws IgniteCheckedException If failed.
-     */
-    private void validate(HadoopConfiguration hadoopCfg) throws IgniteCheckedException {
-        if (ctx.config().isPeerClassLoadingEnabled())
-            throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop
(disable it using " +
-                "IgniteConfiguration.setPeerClassLoadingEnabled()).");
-    }
 }
\ No newline at end of file


Mime
View raw message