Repository: kylin
Updated Branches:
refs/heads/master 948553b9b -> 2f6d8ec06
https://issues.apache.org/jira/browse/KYLIN-3016 StorageCleanup job doesn't clean up all the
legacy fiels in a in Read/Write seperation environment
Signed-off-by: shaofengshi <shaofengshi@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f6d8ec0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f6d8ec0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f6d8ec0
Branch: refs/heads/master
Commit: 2f6d8ec0674947d34f84ae41d175e7f9742f8420
Parents: 948553b
Author: lptong <tlp325@163.com>
Authored: Wed Nov 8 12:11:20 2017 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Fri Nov 17 14:28:44 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/job/StorageCleanupJob.java | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f6d8ec0/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 6d1d44c..d1cb176 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -31,10 +31,12 @@ import javax.annotation.Nullable;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -71,8 +73,8 @@ public class StorageCleanupJob extends AbstractApplication {
protected boolean delete = false;
protected boolean force = false;
- protected static ExecutableManager executableManager = ExecutableManager
- .getInstance(KylinConfig.getInstanceFromEnv());
+ protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig
+ .getInstanceFromEnv());
protected void cleanUnusedHBaseTables() throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -105,15 +107,19 @@ public class StorageCleanupJob extends AbstractApplication {
delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
force = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_FORCE));
cleanUnusedIntermediateHiveTable();
- cleanUnusedHdfsFiles();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) {
+ cleanUnusedHdfsFiles(HBaseConfiguration.create());
+ }
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
+ cleanUnusedHdfsFiles(conf);
cleanUnusedHBaseTables();
}
- private void cleanUnusedHdfsFiles() throws IOException {
- Configuration conf = HadoopUtil.getCurrentConfiguration();
+ private void cleanUnusedHdfsFiles(Configuration conf) throws IOException {
+
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-
FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
// GlobFilter filter = new
|