kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject kylin git commit: https://issues.apache.org/jira/browse/KYLIN-3016 StorageCleanup job doesn't clean up all the legacy fiels in a in Read/Write seperation environment
Date Fri, 17 Nov 2017 06:28:59 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 948553b9b -> 2f6d8ec06


https://issues.apache.org/jira/browse/KYLIN-3016 StorageCleanup job doesn't clean up all the
legacy fiels in a in Read/Write seperation environment

Signed-off-by: shaofengshi <shaofengshi@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f6d8ec0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f6d8ec0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f6d8ec0

Branch: refs/heads/master
Commit: 2f6d8ec0674947d34f84ae41d175e7f9742f8420
Parents: 948553b
Author: lptong <tlp325@163.com>
Authored: Wed Nov 8 12:11:20 2017 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Fri Nov 17 14:28:44 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/rest/job/StorageCleanupJob.java  | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f6d8ec0/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 6d1d44c..d1cb176 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -31,10 +31,12 @@ import javax.annotation.Nullable;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.CliCommandExecutor;
@@ -71,8 +73,8 @@ public class StorageCleanupJob extends AbstractApplication {
 
     protected boolean delete = false;
     protected boolean force = false;
-    protected static ExecutableManager executableManager = ExecutableManager
-            .getInstance(KylinConfig.getInstanceFromEnv());
+    protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig
+            .getInstanceFromEnv());
 
     protected void cleanUnusedHBaseTables() throws IOException {
         KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -105,15 +107,19 @@ public class StorageCleanupJob extends AbstractApplication {
         delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
         force = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_FORCE));
         cleanUnusedIntermediateHiveTable();
-        cleanUnusedHdfsFiles();
+        KylinConfig config = KylinConfig.getInstanceFromEnv();
+        if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) {
+            cleanUnusedHdfsFiles(HBaseConfiguration.create());
+        }
+        Configuration conf = HadoopUtil.getCurrentConfiguration();
+        cleanUnusedHdfsFiles(conf);
         cleanUnusedHBaseTables();
     }
 
-    private void cleanUnusedHdfsFiles() throws IOException {
-        Configuration conf = HadoopUtil.getCurrentConfiguration();
+    private void cleanUnusedHdfsFiles(Configuration conf) throws IOException {
+
         JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
         CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-
         FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
         List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
         // GlobFilter filter = new


Mime
View raw message