kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [28/50] [abbrv] incubator-kylin git commit: minor, fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows
Date Thu, 03 Sep 2015 01:31:43 GMT
minor, fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/572bcbc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/572bcbc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/572bcbc1

Branch: refs/heads/0.7
Commit: 572bcbc1bf9fce7332124063fb119b8f9ac90ae6
Parents: af7f967
Author: Li, Yang <yangli9@ebay.com>
Authored: Mon Aug 24 15:33:29 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Mon Aug 24 15:33:29 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/hadoop/AbstractHadoopJob.java     | 62 ++++++++++----------
 1 file changed, 32 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/572bcbc1/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index f426618..6ad89d6 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -234,35 +234,7 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
         }
 
         dumpResources(kylinConfig, metaDir, dumpList);
-
-        // hadoop distributed cache
-        String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
-        logger.info("HDFS meta dir is: " + hdfsMetaDir);
-        conf.set("tmpfiles", hdfsMetaDir);
-
-    }
-
-    protected void cleanupTempConfFile(Configuration conf) {
-        String tempMetaFileString = conf.get("tmpfiles");
-        logger.info("tempMetaFileString is : " + tempMetaFileString);
-        if (tempMetaFileString != null) {
-            if (tempMetaFileString.startsWith("file://")) {
-                tempMetaFileString = tempMetaFileString.substring("file://".length());
-                File tempMetaFile = new File(tempMetaFileString);
-                if (tempMetaFile.exists()) {
-                    try {
-                        FileUtils.forceDelete(tempMetaFile.getParentFile());
-
-                    } catch (IOException e) {
-                        logger.warn("error when deleting " + tempMetaFile, e);
-                    }
-                } else {
-                    logger.info("" + tempMetaFileString + " does not exist");
-                }
-            } else {
-                logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
-            }
-        }
+        addToHadoopDistCache(conf, metaDir);
     }
 
     protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws
IOException {
@@ -293,9 +265,16 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
         }
 
         dumpResources(kylinConfig, metaDir, dumpList);
+        addToHadoopDistCache(conf, metaDir);
+    }
 
+    private void addToHadoopDistCache(Configuration conf, File metaDir) {
         // hadoop distributed cache
-        String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+        String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+        if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..."
+            hdfsMetaDir = "file://" + hdfsMetaDir;
+        else
+            hdfsMetaDir = "file:///" + hdfsMetaDir;
         logger.info("HDFS meta dir is: " + hdfsMetaDir);
         conf.set("tmpfiles", hdfsMetaDir);
     }
@@ -356,6 +335,29 @@ public abstract class AbstractHadoopJob extends Configured implements
Tool {
         return kylinConfig;
     }
 
+    protected void cleanupTempConfFile(Configuration conf) {
+        String tempMetaFileString = conf.get("tmpfiles");
+        logger.info("tempMetaFileString is : " + tempMetaFileString);
+        if (tempMetaFileString != null) {
+            if (tempMetaFileString.startsWith("file://")) {
+                tempMetaFileString = tempMetaFileString.substring("file://".length());
+                File tempMetaFile = new File(tempMetaFileString);
+                if (tempMetaFile.exists()) {
+                    try {
+                        FileUtils.forceDelete(tempMetaFile.getParentFile());
+
+                    } catch (IOException e) {
+                        logger.warn("error when deleting " + tempMetaFile, e);
+                    }
+                } else {
+                    logger.info("" + tempMetaFileString + " does not exist");
+                }
+            } else {
+                logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
+            }
+        }
+    }
+
     public void kill() throws JobException {
         if (job != null) {
             try {


Mime
View raw message