IGNITE-3383: Hadoop: Improved Hadoop component start logic.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bdeef4c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bdeef4c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bdeef4c0
Branch: refs/heads/ignite-3410
Commit: bdeef4c06a807ca9a4e7f2ed4faa4ca3f0d26f20
Parents: a77e690
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Tue Jun 28 16:33:09 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Tue Jun 28 16:33:09 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteKernal.java | 48 ++++++++++++------
.../processors/hadoop/HadoopNoopProcessor.java | 5 ++
.../hadoop/HadoopProcessorAdapter.java | 7 +++
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopProcessor.java | 52 +++++++++-----------
5 files changed, 68 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 3572293..b6ba4cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -114,7 +114,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor;
+import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.job.GridJobProcessor;
import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor;
@@ -1087,26 +1087,44 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable
{
* Create Hadoop component.
*
* @return Non-null Hadoop component: workable or no-op.
- * @throws IgniteCheckedException if the component is mandatory and cannot be initialized.
+ * @throws IgniteCheckedException If the component is mandatory and cannot be initialized.
*/
- private GridComponent createHadoopComponent() throws IgniteCheckedException {
- GridComponent cmp;
+ private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedException
{
+ boolean mandatory = cfg.getHadoopConfiguration() != null;
- if (cfg.isPeerClassLoadingEnabled()) {
- cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+ if (mandatory) {
+ if (cfg.isPeerClassLoadingEnabled())
+ throw new IgniteCheckedException("Hadoop module cannot be used with peer
class loading enabled " +
+ "(set IgniteConfiguration.peerClassLoadingEnabled to \"false\").");
- if (!(cmp instanceof HadoopNoopProcessor)) {
- U.warn(log, "Hadoop module is found in classpath, but it will not be started
because peer class " +
- "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled
to \"false\" to start " +
- "Hadoop module).");
+ return IgniteComponentType.HADOOP.createIfInClassPath(ctx, true);
+ }
+ else {
+ HadoopProcessorAdapter cmp = null;
- cmp = IgniteComponentType.HADOOP.create(ctx, true/*no-op*/);
+ if (IgniteComponentType.HADOOP.inClassPath() && cfg.isPeerClassLoadingEnabled())
{
+ U.warn(log, "Hadoop module is found in classpath, but will not be started
because peer class " +
+ "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled
to \"false\" if you want " +
+ "to use Hadoop module).");
}
- }
- else
- cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration()
!= null);
+ else {
+ cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+
+ try {
+ cmp.validateEnvironment();
+ }
+ catch (IgniteException | IgniteCheckedException e) {
+ U.quietAndWarn(log, "Hadoop module will not start due to exception: "
+ e.getMessage());
+
+ cmp = null;
+ }
+ }
+
+ if (cmp == null)
+ cmp = IgniteComponentType.HADOOP.create(ctx, true);
- return cmp;
+ return cmp;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
index 72b5050..065cd72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
@@ -74,4 +74,9 @@ public class HadoopNoopProcessor extends HadoopProcessorAdapter {
@Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
return false;
}
+
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // No-oop.
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
index 3d381c3..f914639 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
@@ -94,4 +94,11 @@ public abstract class HadoopProcessorAdapter extends GridProcessorAdapter
{
* @throws IgniteCheckedException If failed.
*/
public abstract boolean kill(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Validate Hadoop environment.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void validateEnvironment() throws IgniteCheckedException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 1c844c4..389de8c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -501,7 +501,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache
{
hadoopUrls = HadoopClasspathUtils.classpathUrls();
}
catch (IOException e) {
- throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.",
e);
+ throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations:
" + e.getMessage(), e);
}
hadoopJars = hadoopUrls;
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index c45953e..7fc7499 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -50,6 +50,8 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
private Hadoop hadoop;
/**
+ * Constructor.
+ *
* @param ctx Kernal context.
*/
public HadoopProcessor(GridKernalContext ctx) {
@@ -70,24 +72,6 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
initializeDefaults(cfg);
- validate(cfg);
-
- try {
- HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
-
- if (loc.home() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
-
- U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
- U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
- U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
- }
- catch (IOException ioe) {
- throw new IgniteCheckedException(ioe);
- }
-
- HadoopClassLoader.hadoopUrls();
-
hctx = new HadoopContext(
ctx,
cfg,
@@ -204,6 +188,26 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
return hctx.jobTracker().killJob(jobId);
}
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // Perform some static checks as early as possible, so that any recoverable exceptions
are thrown here.
+ try {
+ HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
+
+ if (loc.home() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
+
+ U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
+ U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
+ U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
+ }
+ catch (IOException ioe) {
+ throw new IgniteCheckedException(ioe.getMessage(), ioe);
+ }
+
+ HadoopClassLoader.hadoopUrls();
+ }
+
/**
* Initializes default hadoop configuration.
*
@@ -213,16 +217,4 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
if (cfg.getMapReducePlanner() == null)
cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner());
}
-
- /**
- * Validates Grid and Hadoop configuration for correctness.
- *
- * @param hadoopCfg Hadoop configuration.
- * @throws IgniteCheckedException If failed.
- */
- private void validate(HadoopConfiguration hadoopCfg) throws IgniteCheckedException {
- if (ctx.config().isPeerClassLoadingEnabled())
- throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop
(disable it using " +
- "IgniteConfiguration.setPeerClassLoadingEnabled()).");
- }
}
\ No newline at end of file
|