kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1839 bug fix
Date Sun, 06 Nov 2016 07:52:11 GMT
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 0ba460d44 -> 75c981cb3


KYLIN-1839 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75c981cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75c981cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75c981cb

Branch: refs/heads/v1.6.0-rc1
Commit: 75c981cb342b188418052b9768ea129bea26469d
Parents: 0ba460d
Author: Yang Li <liyang@apache.org>
Authored: Sun Nov 6 15:50:49 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Sun Nov 6 15:51:55 2016 +0800

----------------------------------------------------------------------
 .../kylin/engine/mr/common/AbstractHadoopJob.java    | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/75c981cb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 21bb10e..1fd1631 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -295,9 +295,9 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
                     continue;
                 }
                 FileSystem fs;
-                if (hdfs.exists(p)) {
+                if (exists(hdfs, p)) {
                     fs = hdfs;
-                } else if (localfs.exists(p)) {
+                } else if (exists(localfs, p)) {
                     fs = localfs;
                 } else {
                     logger.warn("The directory of kylin dependency '" + fileName + "' does
not exist, skip");
@@ -386,6 +386,15 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
         return classpath;
     }
 
+    private static boolean exists(FileSystem fs, Path p) throws IOException {
+        try {
+            return fs.exists(p);
+        } catch (IllegalArgumentException ex) {
+            // can happen when FS mismatch
+            return false;
+        }
+    }
+
     public static int addInputDirs(String input, Job job) throws IOException {
         int folderNum = addInputDirs(StringSplitter.split(input, ","), job);
         logger.info("Number of added folders:" + folderNum);
@@ -401,7 +410,7 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
                 FileSystem fs = FileSystem.get(job.getConfiguration());
                 Path path = new Path(inp);
 
-                if (!fs.exists(path)) {
+                if (!exists(fs, path)) {
                     logger.warn("Path not exist:" + path.toString());
                     continue;
                 }


Mime
View raw message