kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [1/3] kylin git commit: KYLIN-2059 remove the time threshold on storage cleanup
Date Thu, 06 Oct 2016 06:46:09 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 792d4ee3f -> 95ebc8512


KYLIN-2059 remove the time threshold on storage cleanup

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95ebc851
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95ebc851
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95ebc851

Branch: refs/heads/master
Commit: 95ebc851274953023ba092a249c1ddbbee5cd845
Parents: 334c2e0
Author: shaofengshi <shaofengshi@apache.org>
Authored: Fri Sep 30 09:13:03 2016 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Thu Oct 6 14:44:05 2016 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/KylinConfigBase.java     |  4 ----
 .../kylin/engine/mr/steps/MetadataCleanupJob.java    |  2 +-
 .../kylin/storage/hbase/util/StorageCleanupJob.java  | 15 ++++-----------
 3 files changed, 5 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4d1639b..99c3c5a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -780,10 +780,6 @@ abstract public class KylinConfigBase implements Serializable {
         return Long.parseLong(getOptional("kylin.query.hbase.hconnection.threads.alive.seconds",
"60"));
     }
 
-    public long getStorageCleanupTimeThreshold() {
-        return Long.valueOf(this.getOptional("kylin.storage.cleanup.time.threshold", "172800000"));
//default two days
-    }
-
     public int getAppendDictEntrySize() {
         return Integer.parseInt(getOptional("kylin.dict.append.entry.size", "10000000"));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
index 962697e..f2b1d6b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
@@ -54,7 +54,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
 
     private KylinConfig config = null;
 
-    public static final long TIME_THREADSHOLD = 2 * 24 * 3600 * 1000L; // 2 days
+    public static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
     public static final long TIME_THREADSHOLD_FOR_JOB = 30 * 24 * 3600 * 1000L; // 30 days
 
     /*

http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index dffce36..2c2f11c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -34,7 +34,6 @@ import java.util.regex.Pattern;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -63,10 +62,10 @@ public class StorageCleanupJob extends AbstractApplication {
 
     @SuppressWarnings("static-access")
     protected static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete
the unused storage").create("delete");
-    protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning:
will delete any intermediate hive tables").create("force");
+    protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning:
will delete all kylin intermediate hive tables").create("force");
 
     protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class);
-    public static final int TIME_THRESHOLD_DELETE_HTABLE = 10; // Unit minute
+    public static final int deleteTimeout = 10; // Unit minute
 
     protected boolean delete = false;
     protected boolean force = false;
@@ -74,7 +73,6 @@ public class StorageCleanupJob extends AbstractApplication {
 
     private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
         CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-        long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold();
         // get all kylin hbase tables
         HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
         String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
@@ -82,14 +80,9 @@ public class StorageCleanupJob extends AbstractApplication {
         List<String> allTablesNeedToBeDropped = new ArrayList<String>();
         for (HTableDescriptor desc : tableDescriptors) {
             String host = desc.getValue(IRealizationConstants.HTableTag);
-            String creationTime = desc.getValue(IRealizationConstants.HTableCreationTime);
             if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
{
                 //only take care htables that belongs to self, and created more than 2 days
-                if (StringUtils.isEmpty(creationTime) || (System.currentTimeMillis() - Long.valueOf(creationTime)
> TIME_THREADSHOLD)) {
                     allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
-                } else {
-                    logger.info("Exclude table " + desc.getTableName().getNameAsString()
+ " from drop list, as it is newly created");
-                }
             }
         }
 
@@ -111,9 +104,9 @@ public class StorageCleanupJob extends AbstractApplication {
                 FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin,
htableName));
                 executorService.execute(futureTask);
                 try {
-                    futureTask.get(TIME_THRESHOLD_DELETE_HTABLE, TimeUnit.MINUTES);
+                    futureTask.get(deleteTimeout, TimeUnit.MINUTES);
                 } catch (TimeoutException e) {
-                    logger.warn("It fails to delete htable " + htableName + ", for it cost
more than " + TIME_THRESHOLD_DELETE_HTABLE + " minutes!");
+                    logger.warn("It fails to delete htable " + htableName + ", for it cost
more than " + deleteTimeout + " minutes!");
                     futureTask.cancel(true);
                 } catch (Exception e) {
                     e.printStackTrace();


Mime
View raw message