kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject kylin git commit: code refactor: make all tool CLI subclass of AbstractApplication
Date Thu, 14 Jul 2016 06:49:41 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 49a0a41f0 -> bb1cbda97


code refactor: make all tool CLI subclass of AbstractApplication


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb1cbda9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb1cbda9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb1cbda9

Branch: refs/heads/master
Commit: bb1cbda97ba9278e3ee77f1df593cc533b89dc75
Parents: 49a0a41
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Thu Jul 14 14:49:03 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Thu Jul 14 14:49:03 2016 +0800

----------------------------------------------------------------------
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../engine/mr/common/AbstractHadoopJob.java     |  1 -
 .../mr/steps/RowKeyDistributionCheckerJob.java  |  6 +-
 .../kylin/provision/BuildCubeWithEngine.java    |  6 +-
 .../kylin/provision/BuildCubeWithStream.java    |  6 +-
 .../apache/kylin/rest/service/AdminService.java |  3 +-
 .../storage/hbase/util/CleanHtableCLI.java      | 35 +++++------
 .../kylin/storage/hbase/util/HBaseClean.java    | 53 +++++++---------
 .../hbase/util/HtableAlterMetadataCLI.java      | 53 ++++++++--------
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 66 +++++++++-----------
 .../storage/hbase/util/StorageCleanupJob.java   | 65 ++++++++-----------
 11 files changed, 131 insertions(+), 165 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index ab70007..4dedad1 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -95,7 +95,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent
{
     @Override
     public final ExecuteResult execute(ExecutableContext executableContext) throws ExecuteException
{
 
-        logger.info("Executing >>>>>>>>>>>>>   " +
this.getName() + "   <<<<<<<<<<<<<");
+        logger.info("Executing AbstractExecutable (" + this.getName() + ")");
 
         Preconditions.checkArgument(executableContext instanceof DefaultContext);
         ExecuteResult result = null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index bafe65b..b483dc1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -77,7 +77,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool
{
     protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job
name. For example, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
     protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube
name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
     protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID
of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID);
-    protected static final Option OPTION_II_NAME = OptionBuilder.withArgName(BatchConstants.ARG_II_NAME).hasArg().isRequired(true).withDescription("II
name. For exmaple, some_ii").create(BatchConstants.ARG_II_NAME);
     protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube
segment name").create(BatchConstants.ARG_SEGMENT_NAME);
     protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input
path").create(BatchConstants.ARG_INPUT);
     protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName(BatchConstants.ARG_INPUT_FORMAT).hasArg().isRequired(false).withDescription("Input
format").create(BatchConstants.ARG_INPUT_FORMAT);

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java
index c5a0ff2..fc8d3e7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerJob.java
@@ -39,7 +39,7 @@ import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 public class RowKeyDistributionCheckerJob extends AbstractHadoopJob {
 
     @SuppressWarnings("static-access")
-    protected static final Option rowKeyStatsFilePath = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("rowKeyStatsFilePath").create("rowKeyStatsFilePath");
+    protected static final Option ROW_KEY_STATS_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("rowKeyStatsFilePath").create("rowKeyStatsFilePath");
 
     @Override
     public int run(String[] args) throws Exception {
@@ -49,11 +49,11 @@ public class RowKeyDistributionCheckerJob extends AbstractHadoopJob {
             options.addOption(OPTION_INPUT_PATH);
             options.addOption(OPTION_OUTPUT_PATH);
             options.addOption(OPTION_JOB_NAME);
-            options.addOption(rowKeyStatsFilePath);
+            options.addOption(ROW_KEY_STATS_FILE_PATH);
 
             parseOptions(options, args);
 
-            String statsFilePath = getOptionValue(rowKeyStatsFilePath);
+            String statsFilePath = getOptionValue(ROW_KEY_STATS_FILE_PATH);
 
             // start job
             String jobName = getOptionValue(OPTION_JOB_NAME);

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 9ce2bf2..3d60a3c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -414,8 +413,9 @@ public class BuildCubeWithEngine {
     private int cleanupOldStorage() throws Exception {
         String[] args = { "--delete", "true" };
 
-        int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
-        return exitCode;
+        StorageCleanupJob cli = new StorageCleanupJob();
+        cli.execute(args);
+        return 0;
     }
 
     private void checkHFilesInHBase(CubeSegment segment) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 1655a17..d41dc34 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.util.UUID;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.DateFormat;
@@ -111,8 +110,9 @@ public class BuildCubeWithStream {
     private static int cleanupOldStorage() throws Exception {
         String[] args = { "--delete", "true" };
 
-        int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
-        return exitCode;
+        StorageCleanupJob cli = new StorageCleanupJob();
+        cli.execute(args);
+        return 0;
     }
 
     public void build() throws Exception {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index 6c85898..b92c305 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -25,7 +25,6 @@ import java.util.Properties;
 
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.InternalErrorException;
@@ -101,7 +100,7 @@ public class AdminService extends BasicService {
         StorageCleanupJob job = new StorageCleanupJob();
         String[] args = new String[] { "-delete", "true" };
         try {
-            ToolRunner.run(job, args);
+            job.execute(args);
         } catch (Exception e) {
             throw new InternalErrorException(e.getMessage(), e);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 9e30a06..a150607 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -24,8 +24,8 @@ import org.apache.commons.cli.Options;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.slf4j.Logger;
@@ -33,24 +33,10 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-public class CleanHtableCLI extends AbstractHadoopJob {
+public class CleanHtableCLI extends AbstractApplication {
 
     protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
 
-    @Override
-    public int run(String[] args) throws Exception {
-        Options options = new Options();
-        try {
-
-            clean();
-
-            return 0;
-        } catch (Exception e) {
-            printUsage(options);
-            throw e;
-        }
-    }
-
     private void clean() throws IOException {
         Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
         HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
@@ -71,7 +57,18 @@ public class CleanHtableCLI extends AbstractHadoopJob {
     }
 
     public static void main(String[] args) throws Exception {
-        int exitCode = ToolRunner.run(new CleanHtableCLI(), args);
-        System.exit(exitCode);
+        CleanHtableCLI cli = new CleanHtableCLI();
+        cli.execute(args);
+    }
+
+    @Override
+    protected Options getOptions() {
+        Options options = new Options();
+        return options;
+    }
+
+    @Override
+    protected void execute(OptionsHelper optionsHelper) throws Exception {
+        clean();
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index b6958d6..6749d6c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -19,7 +19,6 @@
 package org.apache.kylin.storage.hbase.util;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.cli.Option;
@@ -28,8 +27,8 @@ import org.apache.commons.cli.Options;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.slf4j.Logger;
@@ -40,7 +39,7 @@ import com.google.common.collect.Lists;
 /**
  * clean hbase tables by tag
  */
-public class HBaseClean extends AbstractHadoopJob {
+public class HBaseClean extends AbstractApplication {
 
     @SuppressWarnings("static-access")
     private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(true).withDescription("actually
delete or not").create("delete");
@@ -49,33 +48,10 @@ public class HBaseClean extends AbstractHadoopJob {
     private static final Option OPTION_TAG = OptionBuilder.withArgName("tag").hasArg().isRequired(true).withDescription("the
tag of HTable").create("tag");
 
     protected static final Logger logger = LoggerFactory.getLogger(HBaseClean.class);
+
     boolean delete = false;
     String tag = null;
 
-    @Override
-    public int run(String[] args) throws Exception {
-        Options options = new Options();
-
-        logger.info("jobs args: " + Arrays.toString(args));
-        try {
-            options.addOption(OPTION_DELETE);
-            options.addOption(OPTION_TAG);
-            parseOptions(options, args);
-
-            logger.info("options: '" + getOptionsAsString() + "'");
-
-            tag = getOptionValue(OPTION_TAG);
-            delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
-
-            cleanUp();
-
-            return 0;
-        } catch (Exception e) {
-            e.printStackTrace(System.err);
-            throw e;
-        }
-    }
-
     private void cleanUp() {
         try {
             // get all kylin hbase tables
@@ -121,7 +97,24 @@ public class HBaseClean extends AbstractHadoopJob {
     }
 
     public static void main(String[] args) throws Exception {
-        int exitCode = ToolRunner.run(new HBaseClean(), args);
-        System.exit(exitCode);
+        HBaseClean cli = new HBaseClean();
+        cli.execute(args);
+    }
+
+    @Override
+    protected Options getOptions() {
+        Options options = new Options();
+        options.addOption(OPTION_DELETE);
+        options.addOption(OPTION_TAG);
+        return options;
+
+    }
+
+    @Override
+    protected void execute(OptionsHelper optionsHelper) throws Exception {
+        tag = optionsHelper.getOptionValue(OPTION_TAG);
+        delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
+
+        cleanUp();
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index 4db183b..ca1a060 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -27,8 +27,9 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,10 +37,11 @@ import org.slf4j.LoggerFactory;
 /**
  */
 @SuppressWarnings("static-access")
-public class HtableAlterMetadataCLI extends AbstractHadoopJob {
+public class HtableAlterMetadataCLI extends AbstractApplication {
 
     private static final Option OPTION_METADATA_KEY = OptionBuilder.withArgName("key").hasArg().isRequired(true).withDescription("The
metadata key").create("key");
     private static final Option OPTION_METADATA_VALUE = OptionBuilder.withArgName("value").hasArg().isRequired(true).withDescription("The
metadata value").create("value");
+    protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_HTABLE_NAME).hasArg().isRequired(true).withDescription("HTable
name").create(BatchConstants.ARG_HTABLE_NAME);
 
     protected static final Logger logger = LoggerFactory.getLogger(HtableAlterMetadataCLI.class);
 
@@ -47,28 +49,6 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob {
     String metadataKey;
     String metadataValue;
 
-    @Override
-    public int run(String[] args) throws Exception {
-        Options options = new Options();
-        try {
-            options.addOption(OPTION_HTABLE_NAME);
-            options.addOption(OPTION_METADATA_KEY);
-            options.addOption(OPTION_METADATA_VALUE);
-
-            parseOptions(options, args);
-            tableName = getOptionValue(OPTION_HTABLE_NAME);
-            metadataKey = getOptionValue(OPTION_METADATA_KEY);
-            metadataValue = getOptionValue(OPTION_METADATA_VALUE);
-
-            alter();
-
-            return 0;
-        } catch (Exception e) {
-            printUsage(options);
-            throw e;
-        }
-    }
-
     private void alter() throws IOException {
         Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
         HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
@@ -82,7 +62,26 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob {
     }
 
     public static void main(String[] args) throws Exception {
-        int exitCode = ToolRunner.run(new HtableAlterMetadataCLI(), args);
-        System.exit(exitCode);
+        HtableAlterMetadataCLI cli = new HtableAlterMetadataCLI();
+        cli.execute(args);
+    }
+
+    @Override
+    protected Options getOptions() {
+        Options options = new Options();
+        options.addOption(OPTION_HTABLE_NAME);
+        options.addOption(OPTION_METADATA_KEY);
+        options.addOption(OPTION_METADATA_VALUE);
+        return options;
+    }
+
+    @Override
+    protected void execute(OptionsHelper optionsHelper) throws Exception {
+        tableName = optionsHelper.getOptionValue(OPTION_HTABLE_NAME);
+        metadataKey = optionsHelper.getOptionValue(OPTION_METADATA_KEY);
+        metadataValue = optionsHelper.getOptionValue(OPTION_METADATA_VALUE);
+
+        alter();
+
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index f0618c9..8ff5b0f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase.util;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
@@ -32,15 +31,15 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  */
-public class OrphanHBaseCleanJob extends AbstractHadoopJob {
+public class OrphanHBaseCleanJob extends AbstractApplication {
 
     @SuppressWarnings("static-access")
     private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete
the unused storage").create("delete");
@@ -52,37 +51,6 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob {
     boolean delete = false;
     Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 
-    @Override
-    public int run(String[] args) throws Exception {
-        Options options = new Options();
-
-        logger.info("jobs args: " + Arrays.toString(args));
-        try {
-            options.addOption(OPTION_DELETE);
-            options.addOption(OPTION_WHITELIST);
-            parseOptions(options, args);
-
-            logger.info("options: '" + getOptionsAsString() + "'");
-            logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
-            delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
-            String[] metastoreWhitelist = getOptionValue(OPTION_WHITELIST).split(",");
-
-            for (String ms : metastoreWhitelist) {
-                logger.info("metadata store in white list: " + ms);
-                metastoreWhitelistSet.add(ms);
-            }
-
-            Configuration conf = HBaseConfiguration.create(getConf());
-
-            cleanUnusedHBaseTables(conf);
-
-            return 0;
-        } catch (Exception e) {
-            e.printStackTrace(System.err);
-            throw e;
-        }
-    }
-
     private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
 
         // get all kylin hbase tables
@@ -128,7 +96,31 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob {
     }
 
     public static void main(String[] args) throws Exception {
-        int exitCode = ToolRunner.run(new OrphanHBaseCleanJob(), args);
-        System.exit(exitCode);
+        OrphanHBaseCleanJob job = new OrphanHBaseCleanJob();
+        job.execute(args);
+    }
+
+    @Override
+    protected Options getOptions() {
+        Options options = new Options();
+        options.addOption(OPTION_DELETE);
+        options.addOption(OPTION_WHITELIST);
+        return options;
+    }
+
+    @Override
+    protected void execute(OptionsHelper optionsHelper) throws Exception {
+        logger.info("options: '" + optionsHelper.getOptionsAsString() + "'");
+        logger.info("delete option value: '" + optionsHelper.getOptionValue(OPTION_DELETE)
+ "'");
+        delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
+        String[] metastoreWhitelist = optionsHelper.getOptionValue(OPTION_WHITELIST).split(",");
+
+        for (String ms : metastoreWhitelist) {
+            logger.info("metadata store in white list: " + ms);
+            metastoreWhitelistSet.add(ms);
+        }
+
+        Configuration conf = HBaseConfiguration.create();
+        cleanUnusedHBaseTables(conf);
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1cbda9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 3f4a6d5..214375a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -22,7 +22,6 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
@@ -42,14 +41,14 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.job.execution.ExecutableState;
@@ -58,50 +57,17 @@ import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class StorageCleanupJob extends AbstractHadoopJob {
+public class StorageCleanupJob extends AbstractApplication {
 
     @SuppressWarnings("static-access")
     private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete
the unused storage").create("delete");
 
     protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class);
-
     public static final int TIME_THRESHOLD_DELETE_HTABLE = 10; // Unit minute
 
     boolean delete = false;
-
     protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
-     */
-    @Override
-    public int run(String[] args) throws Exception {
-        Options options = new Options();
-
-        logger.info("jobs args: " + Arrays.toString(args));
-        try {
-            options.addOption(OPTION_DELETE);
-            parseOptions(options, args);
-
-            logger.info("options: '" + getOptionsAsString() + "'");
-            logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
-            delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
-
-            Configuration conf = HBaseConfiguration.create(getConf());
-
-            cleanUnusedIntermediateHiveTable(conf);
-            cleanUnusedHdfsFiles(conf);
-            cleanUnusedHBaseTables(conf);
-
-            return 0;
-        } catch (Exception e) {
-            printUsage(options);
-            throw e;
-        }
-    }
-
     private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
         CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
         long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold();
@@ -162,6 +128,27 @@ public class StorageCleanupJob extends AbstractHadoopJob {
         hbaseAdmin.close();
     }
 
+    @Override
+    protected Options getOptions() {
+        Options options = new Options();
+        options.addOption(OPTION_DELETE);
+        return options;
+    }
+
+    @Override
+    protected void execute(OptionsHelper optionsHelper) throws Exception {
+        logger.info("options: '" + optionsHelper.getOptionsAsString() + "'");
+        logger.info("delete option value: '" + optionsHelper.getOptionValue(OPTION_DELETE)
+ "'");
+        delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
+
+        Configuration conf = HBaseConfiguration.create();
+
+        cleanUnusedIntermediateHiveTable(conf);
+        cleanUnusedHdfsFiles(conf);
+        cleanUnusedHBaseTables(conf);
+
+    }
+
     class DeleteHTableRunnable implements Callable {
         HBaseAdmin hbaseAdmin;
         String htableName;
@@ -322,7 +309,7 @@ public class StorageCleanupJob extends AbstractHadoopJob {
     }
 
     public static void main(String[] args) throws Exception {
-        int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
-        System.exit(exitCode);
+        StorageCleanupJob cli = new StorageCleanupJob();
+        cli.execute(args);
     }
 }


Mime
View raw message