Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@ignite.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: sboikov@apache.org
To: commits@ignite.apache.org
Date: Thu, 07 Jul 2016 06:01:55 -0000
Message-Id: <682114d9853d4e8b92e769f6f0b531dd@git.apache.org>
In-Reply-To: <078d677a2cf641a7b437be012bf63361@git.apache.org>
References: <078d677a2cf641a7b437be012bf63361@git.apache.org>
Subject: [30/50] [abbrv] ignite git commit: IGNITE-3185: Hadoop: Improved
 classpath resolution logic.
archived-at: Thu, 07 Jul 2016 06:01:31 -0000

IGNITE-3185: Hadoop: Improved classpath resolution logic.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/34798d3a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/34798d3a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/34798d3a

Branch: refs/heads/ignite-1232
Commit: 34798d3ae007c99c4c813d7a1e060f4f145b4ec2
Parents: 9d033e5
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Fri Jul 1 15:39:40 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Wed Jul 6 09:54:18 2016 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopClasspathUtils.java | 160 +++++++++++++------
 .../processors/hadoop/HadoopLocations.java      |  79 +++++++--
 .../processors/hadoop/HadoopProcessor.java      |  12 +-
 3 files changed, 179 insertions(+), 72 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/34798d3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index 986b23e..121fcab 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -33,6 +33,24 @@ import java.util.List;
  * Hadoop classpath utilities.
  */
 public class HadoopClasspathUtils {
+    /** Prefix directory. */
+    public static final String PREFIX = "HADOOP_PREFIX";
+
+    /** Home directory. */
+    public static final String HOME = "HADOOP_HOME";
+
+    /** Home directory. */
+    public static final String COMMON_HOME = "HADOOP_COMMON_HOME";
+
+    /** Home directory. */
+    public static final String HDFS_HOME = "HADOOP_HDFS_HOME";
+
+    /** Home directory. */
+    public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME";
+
+    /** Empty string. */
+    private static final String EMPTY_STR = "";
+
     /**
      * Gets Hadoop class path as list of classpath elements for process.
      *
@@ -80,16 +98,77 @@ public class HadoopClasspathUtils {
     /**
      * Gets Hadoop locations.
      *
-     * @return The Hadoop locations, never null.
+     * @return The locations as determined from the environment.
      */
-    public static HadoopLocations hadoopLocations() throws IOException {
-        final String hadoopHome = systemOrEnv("HADOOP_HOME", systemOrEnv("HADOOP_PREFIX", null));
+    public static HadoopLocations locations() throws IOException {
+        // Query environment.
+        String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR));
+
+        String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR);
+        String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR);
+        String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR);
 
-        String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
-        String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
-        String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+        // If any composite location is defined, use only them.
+        if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) {
+            HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
 
-        return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+            if (res.valid())
+                return res;
+            else
+                throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " +
+                    "either undefined or point to nonexistent directories [" +
+                    "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " +
+                    "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " +
+                    "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]");
+        }
+        else if (!isEmpty(hadoopHome)) {
+            // All further checks will be based on HADOOP_HOME, so check for it's existence.
+            if (!exists(hadoopHome))
+                throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
+                    "variable points to nonexistent directory: " + hadoopHome);
+
+            // Probe Apache Hadoop.
+            HadoopLocations res = new HadoopLocations(
+                hadoopHome,
+                hadoopHome + "/share/hadoop/common",
+                hadoopHome + "/share/hadoop/hdfs",
+                hadoopHome + "/share/hadoop/mapreduce"
+            );
+
+            if (res.valid())
+                return res;
+
+            // Probe CDH.
+            res = new HadoopLocations(
+                hadoopHome,
+                hadoopHome,
+                hadoopHome + "/../hadoop-hdfs",
+                hadoopHome + "/../hadoop-mapreduce"
+            );
+
+            if (res.valid())
+                return res;
+
+            // Probe HDP.
+            res = new HadoopLocations(
+                hadoopHome,
+                hadoopHome,
+                hadoopHome + "/../hadoop-hdfs-client",
+                hadoopHome + "/../hadoop-mapreduce-client"
+            );
+
+            if (res.valid())
+                return res;
+
+            // Failed.
+            throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " +
+                "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome);
+        }
+        else {
+            // Advise to set HADOOP_HOME only as this is preferred way to configure classpath.
+            throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " +
+                "variable and point it to your Hadoop distribution).");
+        }
     }
 
     /**
@@ -99,57 +178,30 @@ public class HadoopClasspathUtils {
      * @throws IOException if a mandatory classpath location is not found.
      */
     private static Collection<SearchDirectory> classpathDirectories() throws IOException {
-        HadoopLocations loc = hadoopLocations();
+        HadoopLocations loc = locations();
 
         Collection<SearchDirectory> res = new ArrayList<>();
 
-        res.add(new SearchDirectory(new File(loc.commonHome(), "lib"), null));
-        res.add(new SearchDirectory(new File(loc.hdfsHome(), "lib"), null));
-        res.add(new SearchDirectory(new File(loc.mapredHome(), "lib"), null));
+        res.add(new SearchDirectory(new File(loc.common(), "lib"), null));
+        res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), null));
+        res.add(new SearchDirectory(new File(loc.mapred(), "lib"), null));
 
-        res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-common-"));
-        res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-auth-"));
+        res.add(new SearchDirectory(new File(loc.common()), "hadoop-common-"));
+        res.add(new SearchDirectory(new File(loc.common()), "hadoop-auth-"));
 
-        res.add(new SearchDirectory(new File(loc.hdfsHome()), "hadoop-hdfs-"));
+        res.add(new SearchDirectory(new File(loc.hdfs()), "hadoop-hdfs-"));
 
-        res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-common"));
-        res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-core"));
+        res.add(new SearchDirectory(new File(loc.mapred()), "hadoop-mapreduce-client-common"));
+        res.add(new SearchDirectory(new File(loc.mapred()), "hadoop-mapreduce-client-core"));
 
         return res;
     }
 
     /**
-     * Resolves a Hadoop location directory.
-     *
-     * @param envVarName Environment variable name. The value denotes the location path.
-     * @param hadoopHome Hadoop home location, may be null.
-     * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
-     * @throws IOException If the value cannot be resolved to an existing directory.
-     */
-    private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
-        throws IOException {
-        String val = systemOrEnv(envVarName, null);
-
-        if (val == null) {
-            // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
-            if (!directoryExists(hadoopHome))
-                throw new IOException("Failed to resolve Hadoop installation location. " +
-                        envVarName + " or HADOOP_HOME environment variable should be set.");
-
-            val = hadoopHome + expHadoopHomeRelativePath;
-        }
-
-        if (!directoryExists(val))
-            throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
-
-        return val;
-    }
-
-    /**
-     * Note that this method does not treat empty value as an absent value.
+     * Get system property or environment variable with the given name.
      *
      * @param name Variable name.
-     * @param dflt Default.
+     * @param dflt Default value.
      * @return Value.
      */
     private static String systemOrEnv(String name, String dflt) {
@@ -158,7 +210,7 @@ public class HadoopClasspathUtils {
         if (res == null)
             res = System.getenv(name);
 
-        return res == null ? dflt : res;
+        return res != null ? res : dflt;
     }
 
     /**
@@ -167,7 +219,7 @@ public class HadoopClasspathUtils {
      * @param path The directory path.
      * @return {@code True} if the given path denotes an existing directory.
      */
-    private static boolean directoryExists(String path) {
+    public static boolean exists(String path) {
         if (path == null)
             return false;
 
@@ -177,6 +229,16 @@ public class HadoopClasspathUtils {
     }
 
     /**
+     * Check if string is empty.
+     *
+     * @param val Value.
+     * @return {@code True} if empty.
+     */
+    private static boolean isEmpty(String val) {
+        return val == null || val.isEmpty();
+    }
+
+    /**
      * Simple pair-like structure to hold directory name and a mask assigned to it.
      */
     private static class SearchDirectory {
@@ -196,7 +258,7 @@ public class HadoopClasspathUtils {
             this.dir = dir;
             this.filter = filter;
 
-            if (!directoryExists(dir.getAbsolutePath()))
+            if (!exists(dir.getAbsolutePath()))
                 throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
         }
 
@@ -230,4 +292,4 @@ public class HadoopClasspathUtils {
             return filter != null;
         }
     }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/34798d3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
index b3e2d12..a90007f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -25,27 +25,42 @@ public class HadoopLocations {
     private final String home;
 
     /** Common home. */
-    private final String commonHome;
+    private final String common;
 
     /** HDFS home. */
-    private final String hdfsHome;
+    private final String hdfs;
 
     /** Mapred home. */
-    private final String mapredHome;
+    private final String mapred;
+
+    /** Whether common home exists. */
+    private final boolean commonExists;
+
+    /** Whether HDFS home exists. */
+    private final boolean hdfsExists;
+
+    /** Whether mapred home exists. */
+    private final boolean mapredExists;
 
     /**
      * Constructor.
      *
      * @param home Hadoop home.
-     * @param commonHome Common home.
-     * @param hdfsHome HDFS home.
-     * @param mapredHome Mapred home.
+     * @param common Common home.
+     * @param hdfs HDFS home.
+     * @param mapred Mapred home.
      */
-    public HadoopLocations(String home, String commonHome, String hdfsHome, String mapredHome) {
+    public HadoopLocations(String home, String common, String hdfs, String mapred) {
+        assert common != null && hdfs != null && mapred != null;
+
         this.home = home;
-        this.commonHome = commonHome;
-        this.hdfsHome = hdfsHome;
-        this.mapredHome = mapredHome;
+        this.common = common;
+        this.hdfs = hdfs;
+        this.mapred = mapred;
+
+        commonExists = HadoopClasspathUtils.exists(common);
+        hdfsExists = HadoopClasspathUtils.exists(hdfs);
+        mapredExists = HadoopClasspathUtils.exists(mapred);
     }
 
     /**
@@ -58,21 +73,51 @@ public class HadoopLocations {
     /**
      * @return Common home.
      */
-    public String commonHome() {
-        return commonHome;
+    public String common() {
+        return common;
     }
 
     /**
      * @return HDFS home.
      */
-    public String hdfsHome() {
-        return hdfsHome;
+    public String hdfs() {
+        return hdfs;
     }
 
     /**
      * @return Mapred home.
      */
-    public String mapredHome() {
-        return mapredHome;
+    public String mapred() {
+        return mapred;
+    }
+
+    /**
+     * @return Whether common home exists.
+     */
+    public boolean commonExists() {
+        return commonExists;
+    }
+
+    /**
+     * @return Whether HDFS home exists.
+     */
+    public boolean hdfsExists() {
+        return hdfsExists;
+    }
+
+    /**
+     * @return Whether mapred home exists.
+     */
+    public boolean mapredExists() {
+        return mapredExists;
+    }
+
+    /**
+     * Whether all required directories exists.
+     *
+     * @return {@code True} if exists.
+     */
+    public boolean valid() {
+        return commonExists && hdfsExists && mapredExists;
     }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/34798d3a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index bb10565..b9c20c3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.hadoop.jobtracker.HadoopJobTracker;
 import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffle;
 import org.apache.ignite.internal.processors.hadoop.taskexecutor.HadoopEmbeddedTaskExecutor;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -190,14 +191,13 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
     @Override public void validateEnvironment() throws IgniteCheckedException {
         // Perform some static checks as early as possible, so that any recoverable exceptions are thrown here.
         try {
-            HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
+            HadoopLocations loc = HadoopClasspathUtils.locations();
 
-            if (loc.home() != null)
-                U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
+            if (!F.isEmpty(loc.home()))
+                U.quietAndInfo(log, HadoopClasspathUtils.HOME + " is set to " + loc.home());
 
-            U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
-            U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
-            U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
+            U.quietAndInfo(log, "Resolved Hadoop classpath locations: " + loc.common() + ", " + loc.hdfs() + ", " +
+                loc.mapred());
         }
         catch (IOException ioe) {
             throw new IgniteCheckedException(ioe.getMessage(), ioe);