kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [01/18] kylin git commit: KYLIN-2166 Unclosed HBaseAdmin in StorageCleanupJob#cleanUnusedHBaseTables [Forced Update!]
Date Tue, 08 Nov 2016 16:15:39 GMT
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2006 db9910568 -> aea46d7ce (forced update)


KYLIN-2166 Unclosed HBaseAdmin in StorageCleanupJob#cleanUnusedHBaseTables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc1866a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc1866a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc1866a8

Branch: refs/heads/KYLIN-2006
Commit: dc1866a8f21956acea6d489e37a731e87faf4502
Parents: 7211d92
Author: lidongsjtu <lidong@apache.org>
Authored: Mon Nov 7 23:59:15 2016 +0800
Committer: lidongsjtu <lidong@apache.org>
Committed: Mon Nov 7 23:59:55 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/tool/StorageCleanupJob.java    | 81 ++++++++++----------
 1 file changed, 40 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc1866a8/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 3f82e94..4252e74 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -82,55 +82,54 @@ public class StorageCleanupJob extends AbstractApplication {
     private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
         CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
         // get all kylin hbase tables
-        HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
-        String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
-        HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
-        List<String> allTablesNeedToBeDropped = new ArrayList<String>();
-        for (HTableDescriptor desc : tableDescriptors) {
-            String host = desc.getValue(IRealizationConstants.HTableTag);
-            if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
{
-                //only take care htables that belongs to self, and created more than 2 days
-                allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
+        try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
+            String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+            HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix +
".*");
+            List<String> allTablesNeedToBeDropped = new ArrayList<String>();
+            for (HTableDescriptor desc : tableDescriptors) {
+                String host = desc.getValue(IRealizationConstants.HTableTag);
+                if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
{
+                    //only take care htables that belongs to self, and created more than
2 days
+                    allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
+                }
             }
-        }
 
-        // remove every segment htable from drop list
-        for (CubeInstance cube : cubeMgr.listAllCubes()) {
-            for (CubeSegment seg : cube.getSegments()) {
-                String tablename = seg.getStorageLocationIdentifier();
-                if (allTablesNeedToBeDropped.contains(tablename)) {
-                    allTablesNeedToBeDropped.remove(tablename);
-                    logger.info("Exclude table " + tablename + " from drop list, as the table
belongs to cube " + cube.getName() + " with status " + cube.getStatus());
+            // remove every segment htable from drop list
+            for (CubeInstance cube : cubeMgr.listAllCubes()) {
+                for (CubeSegment seg : cube.getSegments()) {
+                    String tablename = seg.getStorageLocationIdentifier();
+                    if (allTablesNeedToBeDropped.contains(tablename)) {
+                        allTablesNeedToBeDropped.remove(tablename);
+                        logger.info("Exclude table " + tablename + " from drop list, as the
table belongs to cube " + cube.getName() + " with status " + cube.getStatus());
+                    }
                 }
             }
-        }
 
-        if (delete == true) {
-            // drop tables
-            ExecutorService executorService = Executors.newSingleThreadExecutor();
-            for (String htableName : allTablesNeedToBeDropped) {
-                FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin,
htableName));
-                executorService.execute(futureTask);
-                try {
-                    futureTask.get(deleteTimeout, TimeUnit.MINUTES);
-                } catch (TimeoutException e) {
-                    logger.warn("It fails to delete htable " + htableName + ", for it cost
more than " + deleteTimeout + " minutes!");
-                    futureTask.cancel(true);
-                } catch (Exception e) {
-                    e.printStackTrace();
-                    futureTask.cancel(true);
+            if (delete == true) {
+                // drop tables
+                ExecutorService executorService = Executors.newSingleThreadExecutor();
+                for (String htableName : allTablesNeedToBeDropped) {
+                    FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin,
htableName));
+                    executorService.execute(futureTask);
+                    try {
+                        futureTask.get(deleteTimeout, TimeUnit.MINUTES);
+                    } catch (TimeoutException e) {
+                        logger.warn("It fails to delete htable " + htableName + ", for it
cost more than " + deleteTimeout + " minutes!");
+                        futureTask.cancel(true);
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                        futureTask.cancel(true);
+                    }
                 }
+                executorService.shutdown();
+            } else {
+                System.out.println("--------------- Tables To Be Dropped ---------------");
+                for (String htableName : allTablesNeedToBeDropped) {
+                    System.out.println(htableName);
+                }
+                System.out.println("----------------------------------------------------");
             }
-            executorService.shutdown();
-        } else {
-            System.out.println("--------------- Tables To Be Dropped ---------------");
-            for (String htableName : allTablesNeedToBeDropped) {
-                System.out.println(htableName);
-            }
-            System.out.println("----------------------------------------------------");
         }
-
-        hbaseAdmin.close();
     }
 
     @Override


Mime
View raw message