Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@ignite.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: vozerov@apache.org
To: commits@ignite.apache.org
Date: Fri, 01 Jul 2016 12:10:57 -0000
Message-Id: <2e65180ee2a441d48f2cea2310feb3bf@git.apache.org>
In-Reply-To: <916205a14e4d4e05bf9b06a8a2276d6b@git.apache.org>
References: <916205a14e4d4e05bf9b06a8a2276d6b@git.apache.org>
Subject: [14/27] ignite git commit: IGNITE-3185
archived-at: Fri, 01 Jul 2016 12:10:47 -0000

IGNITE-3185


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c4017097
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c4017097
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c4017097

Branch: refs/heads/ignite-3185
Commit: c4017097bd443069090626acba4772de01051f98
Parents: 9cc1607
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Fri Jun 24 17:26:32 2016 +0300
Committer: iveselovskiy <iveselovskiy@gridgain.com>
Committed: Fri Jun 24 17:26:32 2016 +0300

----------------------------------------------------------------------
 .../processors/hadoop/HadoopClasspathUtils.java | 148 ++++++++++++++-----
 1 file changed, 115 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/c4017097/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
index bb7ccd3..acf19bd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -163,21 +163,130 @@ public class HadoopClasspathUtils {
             this.hdfs = hdfs;
             this.mapred = mapred;
         }
+
+        /**
+         * Answers if all the base directories are defined.
+         *
+         * @return 'true' if "common", "hdfs", and "mapred" directories are defined.
+         */
+        public boolean isDefined() {
+            return common != null && hdfs != null && mapred != null;
+        }
+
+        /**
+         * Answers if all the base directories exist.
+         *
+         * @return 'true' if "common", "hdfs", and "mapred" directories do exist.
+         */
+        public boolean exists() {
+            return isExistingDirectory(common)
+                && isExistingDirectory(hdfs)
+                && isExistingDirectory(mapred);
+        }
+
+        /**
+         * Checks if all the base directories exist.
+         *
+         * @return this reference.
+         * @throws IOException if any of the base directories does not exist.
+         */
+        public HadoopLocations existsOrException() throws IOException {
+            if (!isExistingDirectory(common))
+                throw new IOException("Failed to resolve Hadoop installation location. HADOOP_COMMON_HOME " +
+                    "or HADOOP_HOME environment variable should be set.");
+
+            if (!isExistingDirectory(hdfs))
+                throw new IOException("Failed to resolve Hadoop installation location. HADOOP_HDFS_HOME " +
+                    "or HADOOP_HOME environment variable should be set.");
+
+            if (!isExistingDirectory(mapred))
+                throw new IOException("Failed to resolve Hadoop installation location. HADOOP_MAPRED_HOME " +
+                    "or HADOOP_HOME environment variable should be set.");
+
+            return this;
+        }
+    }
+
+    /**
+     * Gets locations from the environment.
+     *
+     * @return The locations as determined from the environment.
+     */
+    private static HadoopLocations getEnvHadoopLocations() {
+        return new HadoopLocations(
+            hadoopHome(),
+            getEnv("HADOOP_COMMON_HOME", null),
+            getEnv("HADOOP_HDFS_HOME", null),
+            getEnv("HADOOP_MAPRED_HOME", null)
+        );
+    }
+
+    /**
+     * Gets locations assuming Apache Hadoop distribution layout.
+     *
+     * @return The locations as for Apache distribution.
+     */
+    private static HadoopLocations getApacheHadoopLocations(String hadoopHome) {
+        return new HadoopLocations(hadoopHome,
+            hadoopHome + "/share/hadoop/common",
+            hadoopHome + "/share/hadoop/hdfs",
+            hadoopHome + "/share/hadoop/mapreduce");
+    }
+
+    /** HDP Hadoop locations. */
+    private static final HadoopLocations HDP_HADOOP_LOCATIONS = new HadoopLocations(
+        "/usr/hdp/current/hadoop-client",
+        "/usr/hdp/current/hadoop-client",
+        "/usr/hdp/current/hadoop-hdfs-client/",
+        "/usr/hdp/current/hadoop-mapreduce-client/");
+
+    /**
+     * HDP locations relative to an arbitrary Hadoop home.
+     *
+     * @param hadoopHome The hadoop home.
+     * @return The locations.
+     */
+    private static HadoopLocations getHdpLocationsRelative(String hadoopHome) {
+        return new HadoopLocations(hadoopHome, hadoopHome,
+            hadoopHome + "/../hadoop-hdfs-client/",
+            hadoopHome + "/../hadoop-mapreduce-client/");
     }
 
     /**
-     * Gets Hadoop locations.
+     * Gets the existing Hadoop locations, if any.
      *
-     * @return The Hadoop locations, never null.
+     * @return Existing Hadoop locations.
+     * @throws IOException If no existing location found.
      */
     public static HadoopLocations getHadoopLocations() throws IOException {
+        // 1. Try locations defined in System properties or environment:
+        HadoopLocations loc = getEnvHadoopLocations();
+
+        if (loc.isDefined())
+            return loc.existsOrException();
+
         final String hadoopHome = hadoopHome();
 
-        String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
-        String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
-        String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+        if (hadoopHome != null) {
+            // If home is defined, it must exist:
+            if (!isExistingDirectory(hadoopHome))
+                throw new IOException("HADOOP_HOME location is not an existing readable directory. [dir="
+                    + hadoopHome + ']');
+
+            // 2. Try Apache Hadoop locations defined relative to HADOOP_HOME:
+            loc = getApacheHadoopLocations(hadoopHome);
 
-        return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+            if (loc.exists())
+                return loc;
+
+            // 3. Try HDP Hadoop locations defined relative to HADOOP_HOME:
+            loc = getHdpLocationsRelative(hadoopHome);
+
+            return loc.existsOrException();
+        }
+
+        // 4. Try absolute HDP (Hortonworks) location:
+        return HDP_HADOOP_LOCATIONS.existsOrException();
     }
 
     /**
@@ -229,33 +338,6 @@ public class HadoopClasspathUtils {
     }
 
     /**
-     * Resolves a Hadoop location directory.
-     *
-     * @param envVarName Environment variable name. The value denotes the location path.
-     * @param hadoopHome Hadoop home location, may be null.
-     * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
-     * @throws IOException If the value cannot be resolved to an existing directory.
-     */
-    private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
-        throws IOException {
-        String val = getEnv(envVarName, null);
-
-        if (val == null) {
-            // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
-            if (!isExistingDirectory(hadoopHome))
-                throw new IOException("Failed to resolve Hadoop installation location. " +
-                        envVarName + " or HADOOP_HOME environment variable should be set.");
-
-            val = hadoopHome + expHadoopHomeRelativePath;
-        }
-
-        if (!isExistingDirectory(val))
-            throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
-
-        return val;
-    }
-
-    /**
      * Note that this method does not treat empty value as an absent value.
      *
      * @param name Variable name.