Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@ignite.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: vozerov@apache.org
To: commits@ignite.apache.org
Date: Fri, 01 Jul 2016 12:10:44 -0000
Message-Id: <916205a14e4d4e05bf9b06a8a2276d6b@git.apache.org>
Subject: [01/27] ignite git commit: 3184 : new java impl
archived-at: Fri, 01 Jul 2016 12:10:47 -0000

Repository: ignite
Updated Branches:
  refs/heads/ignite-3185 [created] 084abddb2


3184 : new java impl


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0c7b08c8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0c7b08c8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0c7b08c8

Branch: refs/heads/ignite-3185
Commit: 0c7b08c8f48a1d9f4f2610ae15c925f591900856
Parents: 8a14e47
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Fri Jun 17 22:34:08 2016 +0300
Committer: iveselovskiy <iveselovskiy@gridgain.com>
Committed: Fri Jun 17 22:34:08 2016 +0300

----------------------------------------------------------------------
 bin/include/setenv.sh                           |  13 +-
 .../ignite/internal/util/IgniteUtils.java       |   3 +-
 .../processors/hadoop/HadoopClassLoader.java    |  88 ++-----
 .../processors/hadoop/HadoopClasspathMain.java  | 244 +++++++++++++++++++
 .../processors/hadoop/HadoopProcessor.java      |   2 +-
 5 files changed, 275 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 74b6608..2b17253 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -72,9 +72,16 @@ if [ "${USER_LIBS}" != "" ]; then
 fi
 
 if [ "${HADOOP_EDITION}" == "1" ]; then
-    . "${SCRIPTS_HOME}"/include/hadoop-classpath.sh
+    #. "${SCRIPTS_HOME}"/include/hadoop-classpath.sh
+    IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain )
 
-    if [ "${IGNITE_HADOOP_CLASSPATH}" != "" ]; then
-        IGNITE_LIBS=${IGNITE_LIBS}${SEP}$IGNITE_HADOOP_CLASSPATH
+    # TODO: DEBUG:
+    echo "IGNITE_HADOOP_CLASSPATH=[${IGNITE_HADOOP_CLASSPATH}]"
+
+    if [ -z "${IGNITE_HADOOP_CLASSPATH}" ]; then
+        # Error already reported.
+        exit 1
+    else
+        IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
     fi
 fi

http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index faf026b..c28e7bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4049,7 +4049,8 @@ public abstract class IgniteUtils {
     public static void quietAndWarn(IgniteLogger log, Object msg, Object shortMsg) {
         warn(log, msg);
 
-        if (log.isQuiet())
+        if (log != null
+            && log.isQuiet())
             quiet(false, shortMsg);
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 4448b2d..8831137 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -17,11 +17,8 @@
 
 package org.apache.ignite.internal.processors.hadoop;
 
-import java.io.File;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
@@ -57,6 +54,7 @@ import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.Remapper;
 import org.objectweb.asm.commons.RemappingClassAdapter;
+import static org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain.*;
 
 /**
  * Class loader allowing explicitly load classes without delegation to parent class loader.
@@ -171,9 +169,9 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
                 ldr = ldr.getParent();
             }
         }
-        catch (Exception e) {
+        catch (Throwable t) {
             U.quietAndWarn(null, "Failed to initialize Hadoop native library " +
-                "(native Hadoop methods might not work properly): " + e);
+                "(native Hadoop methods might not work properly): " + t);
         }
     }
 
@@ -439,39 +437,19 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
         return hasDot;
     }
 
-    /**
-     * @param name Variable name.
-     * @param dflt Default.
-     * @return Value.
-     */
-    private static String getEnv(String name, String dflt) {
-        String res = System.getProperty(name);
-
-        if (F.isEmpty(res))
-            res = System.getenv(name);
-
-        return F.isEmpty(res) ? dflt : res;
-    }
-
-    /**
-     * @param res Result.
-     * @param dir Directory.
-     * @param startsWith Starts with prefix.
-     * @throws MalformedURLException If failed.
-     */
-    private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws Exception {
-        File[] files = dir.listFiles(new FilenameFilter() {
-            @Override public boolean accept(File dir, String name) {
-                return startsWith == null || name.startsWith(startsWith);
-            }
-        });
-
-        if (files == null)
-            throw new IOException("Path is not a directory: " + dir);
-
-        for (File file : files)
-            res.add(file.toURI().toURL());
-    }
+//    /**
+//     * @param name Variable name.
+//     * @param dflt Default.
+//     * @return Value.
+//     */
+//    private static String getEnv(String name, String dflt) {
+//        String res = System.getProperty(name);
+//
+//        if (F.isEmpty(res))
+//            res = System.getenv(name);
+//
+//        return F.isEmpty(res) ? dflt : res;
+//    }
 
     /**
      * @param urls URLs.
@@ -499,13 +477,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
     }
 
     /**
-     * @return HADOOP_HOME Variable.
-     */
-    @Nullable public static String hadoopHome() {
-        return getEnv("HADOOP_PREFIX", getEnv("HADOOP_HOME", null));
-    }
-
-    /**
      * @return Collection of jar URLs.
      * @throws IgniteCheckedException If failed.
      */
@@ -521,33 +492,10 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
             if (hadoopUrls != null)
                 return hadoopUrls;
 
-            hadoopUrls = new ArrayList<>();
-
-            String hadoopPrefix = hadoopHome();
-
-            if (F.isEmpty(hadoopPrefix))
-                throw new IgniteCheckedException("Failed resolve Hadoop installation location. Either HADOOP_PREFIX or " +
-                    "HADOOP_HOME environment variables must be set.");
-
-            String commonHome = getEnv("HADOOP_COMMON_HOME", hadoopPrefix + "/share/hadoop/common");
-            String hdfsHome = getEnv("HADOOP_HDFS_HOME", hadoopPrefix + "/share/hadoop/hdfs");
-            String mapredHome = getEnv("HADOOP_MAPRED_HOME", hadoopPrefix + "/share/hadoop/mapreduce");
-
             try {
-                addUrls(hadoopUrls, new File(commonHome + "/lib"), null);
-                addUrls(hadoopUrls, new File(hdfsHome + "/lib"), null);
-                addUrls(hadoopUrls, new File(mapredHome + "/lib"), null);
-
-                addUrls(hadoopUrls, new File(hdfsHome), "hadoop-hdfs-");
-
-                addUrls(hadoopUrls, new File(commonHome), "hadoop-common-");
-                addUrls(hadoopUrls, new File(commonHome), "hadoop-auth-");
-                addUrls(hadoopUrls, new File(commonHome + "/lib"), "hadoop-auth-");
-
-                addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-common");
-                addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-core");
+                hadoopUrls = getAsUrlList();
             }
-            catch (Exception e) {
+            catch (IOException e) {
                 throw new IgniteCheckedException(e);
             }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
new file mode 100644
index 0000000..a83946f
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -0,0 +1,244 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+import static java.lang.System.out;
+import static java.lang.System.err;
+
+/**
+ * Main class to compose Hadoop classpath depending on the environment.
+ * Note that this class should not depend on any classes or libraries except the JDK default runtime.
+ */
+public class HadoopClasspathMain {
+    /**
+     *
+     * @param args
+     */
+    public static void main(String[] args) throws Exception {
+        final char sep = File.pathSeparatorChar;
+
+        List<String> cp = getAsProcessClasspath();
+
+        for (String s: cp) {
+            if (s != null && s.length() > 0) {
+                out.print(s);
+                out.print(sep);
+            }
+        }
+
+        out.println();
+    }
+
+    /**
+     *
+     * @return
+     */
+    private static List<String> getAsProcessClasspath() throws IOException {
+        Collection<DirAndMask> dams = getClasspathBaseDirectories();
+
+        List<String> list = new ArrayList<>(32);
+
+        for (DirAndMask dam: dams)
+            addAsJavaProcessClasspathElement(list, dam.file, dam.mask);
+
+        // Sort the classpath elements to make it more reliable.
+        Collections.sort(list);
+
+        return list;
+    }
+
+    /**
+     *
+     * @return
+     * @throws IOException
+     */
+    public static List<URL> getAsUrlList() throws IOException {
+        Collection<DirAndMask> dams = getClasspathBaseDirectories();
+
+        List<URL> list = new ArrayList<>(32);
+
+        for (DirAndMask dam: dams)
+            // Note that this procedure does not use '*' classpath patterns,
+            // but adds all the children explicitly:
+            addUrls(list, dam.file, dam.mask);
+
+        Collections.sort(list, new Comparator<URL>() {
+            @Override public int compare(URL u1, URL u2) {
+                String s1 = String.valueOf(u1);
+                String s2 = String.valueOf(u2);
+
+                return s1.compareTo(s2);
+            }
+        });
+
+        for (URL u: list)
+            err.println(u);
+
+        return list;
+    }
+
+    /**
+     * @param res Result.
+     * @param dir Directory.
+     * @param startsWith Starts with prefix.
+     * @throws MalformedURLException If failed.
+     */
+    private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws IOException {
+        File[] files = dir.listFiles(new FilenameFilter() {
+            @Override public boolean accept(File dir, String name) {
+                return startsWith == null || name.startsWith(startsWith);
+            }
+        });
+
+        if (files == null)
+            throw new IOException("Path is not a directory: " + dir);
+
+        for (File file : files)
+            res.add(file.toURI().toURL());
+    }
+
+
+    /**
+     * @param res Result.
+     * @param dir Directory.
+     * @param startsWith Starts with prefix.
+     * @throws MalformedURLException If failed.
+     */
+    private static void addAsJavaProcessClasspathElement(Collection<String> res, File dir, final String startsWith) throws IOException {
+        if (!dir.exists() || !dir.isDirectory() || !dir.canRead())
+            throw new IOException("Path is not an existing readable directory: " + dir);
+
+        if (startsWith == null)
+            res.add(dir.getAbsolutePath() + File.separator + '*');
+        else {
+            File[] files = dir.listFiles(new FilenameFilter() {
+                @Override public boolean accept(File dir, String name) {
+                    return name.startsWith(startsWith);
+                }
+            });
+
+            if (files == null)
+                throw new IOException("Path is not a directory: " + dir);
+
+            for (File file : files)
+                res.add(file.getAbsolutePath());
+        }
+    }
+
+    /**
+     * @return HADOOP_HOME Variable.
+     */
+    public static String hadoopHome() {
+        String prefix = getEnv("HADOOP_PREFIX", null);
+
+        return getEnv("HADOOP_HOME", prefix);
+    }
+
+    /**
+     *
+     * @return
+     * @throws FileNotFoundException
+     */
+    public static Collection<DirAndMask> getClasspathBaseDirectories() throws FileNotFoundException {
+        final String hadoopHome = hadoopHome();
+
+        String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
+        String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
+        String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+
+        Collection<DirAndMask> c = new ArrayList<>();
+
+        c.add(new DirAndMask(new File(commonHome, "lib"), null));
+        c.add(new DirAndMask(new File(hdfsHome, "lib"), null));
+        c.add(new DirAndMask(new File(mapredHome, "lib"), null));
+
+        c.add(new DirAndMask(new File(commonHome), "hadoop-common-"));
+        c.add(new DirAndMask(new File(commonHome), "hadoop-auth-"));
+        c.add(new DirAndMask(new File(commonHome, "lib"), "hadoop-auth-"));
+
+        c.add(new DirAndMask(new File(hdfsHome), "hadoop-hdfs-"));
+
+        c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-common"));
+        c.add(new DirAndMask(new File(mapredHome), "hadoop-mapreduce-client-core"));
+
+        return c;
+    }
+
+    public static class DirAndMask {
+        DirAndMask(File f, String m) {
+            file = f;
+            mask = m;
+        }
+        public final File file;
+        public final String mask;
+    }
+
+    /**
+     * Checks if the variable is empty.
+     *
+     * @param envVarName Environment variable name.
+     * @param hadoopHome The current value.
+     * @param expHadoopHomeRelativePath The path relative to Hadoop home.
+     * @throws FileNotFoundException If the value is empty.
+     */
+    private static String resolveLocation(String envVarName, String hadoopHome,
+        String expHadoopHomeRelativePath) throws FileNotFoundException {
+        String val = getEnv(envVarName, null);
+
+        if (val == null) {
+            // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
+            if (!isExistingDirectory(hadoopHome))
+                throw new FileNotFoundException("Failed to resolve Hadoop installation location. " +
+                        envVarName + " or HADOOP_HOME environment variable should be set.");
+
+            val = hadoopHome + expHadoopHomeRelativePath;
+        }
+
+        if (!isExistingDirectory(val))
+            throw new FileNotFoundException("Failed to resolve Hadoop location. [path=" + val + ']');
+
+        err.println(envVarName + " resolved to " + val);
+
+        return val;
+    }
+
+    /**
+     * Note that this method does not treat empty value as an absent value.
+     *
+     * @param name Variable name.
+     * @param dflt Default.
+     * @return Value.
+     */
+    private static String getEnv(String name, String dflt) {
+        String res = System.getProperty(name);
+
+        if (res == null)
+            res = System.getenv(name);
+
+        return res == null ? dflt : res;
+    }
+
+    /**
+     * Answers if the given path denotes existing directory.
+     *
+     * @param path The directory path.
+     * @return 'true' if the given path denotes an existing directory.
+     */
+    private static boolean isExistingDirectory(String path) {
+        if (path == null)
+            return false;
+
+        Path p = Paths.get(path);
+
+        return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/0c7b08c8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index 7292c78..a502c3e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -33,8 +33,8 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
-import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopHome;
 import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopUrls;
+import static org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain.hadoopHome;
 
 /**
  * Hadoop processor.