kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [2/2] kylin git commit: KYLIN_2752 code clean up
Date Sun, 17 Sep 2017 09:38:59 GMT
KYLIN_2752 code clean up


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/876811f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/876811f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/876811f2

Branch: refs/heads/master
Commit: 876811f2312981b99344cd700e1300a0462ec124
Parents: c9a855f
Author: Li Yang <liyang@apache.org>
Authored: Sun Sep 17 16:21:12 2017 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Sun Sep 17 16:28:19 2017 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/cube/CubeManagerTest.java     | 12 ++++++++----
 .../metadata/realization/IRealizationConstants.java     |  4 ----
 examples/test_case_data/localmeta/kylin.properties      |  6 ------
 .../apache/kylin/rest/job/StorageCleanJobHbaseUtil.java |  7 ++++---
 .../org/apache/kylin/storage/hbase/util/HBaseClean.java |  5 +++--
 .../org/apache/kylin/storage/hbase/util/HBaseUsage.java |  5 +++--
 .../kylin/storage/hbase/util/OrphanHBaseCleanJob.java   |  5 +++--
 .../kylin/storage/hbase/util/StorageCleanupJob.java     |  9 +++++----
 8 files changed, 26 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index da27032..0188904 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -307,10 +307,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
 
     @Test
     public void testGetCubeNameWithNamespace() {
-        CubeManager mgr = CubeManager.getInstance(getTestConfig());
-        String tablename = mgr.generateStorageLocation();
-        System.out.println(tablename);
-        assertTrue(tablename.startsWith("HELLO:WORLD"));
+        System.setProperty("kylin.storage.hbase.table-name-prefix", "HELLO_");
+        try {
+            CubeManager mgr = CubeManager.getInstance(getTestConfig());
+            String tablename = mgr.generateStorageLocation();
+            assertTrue(tablename.startsWith("HELLO_"));
+        } finally {
+            System.clearProperty("kylin.storage.hbase.table-name-prefix");
+        }
     }
 
     public CubeDescManager getCubeDescManager() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 2a3b01e..2e4f400 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -22,10 +22,6 @@ package org.apache.kylin.metadata.realization;
  */
 public class IRealizationConstants {
 
-    public final static String SharedHbaseStorageLocationPrefix = "KYLIN_";
-    public final static String CubeHbaseStorageLocationPrefix = "KYLIN_";
-    public final static String IIHbaseStorageLocationPrefix = "KYLIN_II_";
-
     /**
      * For each cube htable, we leverage htable's metadata to keep track of
      * which kylin server(represented by its kylin_metadata prefix) owns this htable

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 48e2856..c7dda3f 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -39,12 +39,6 @@ kylin.metadata.url=
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 
-# The prefix for cube table
-kylin.storage.hbase.tableprefix=WORLD_
-
-# The namespace for hbase storage
-kylin.storage.hbase.namespace=HELLO
-
 # Working folder in HDFS, make sure user has the right access to the hdfs directory
 kylin.env.hdfs-working-dir=/kylin
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 3728ea1..aceebb4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -46,15 +46,16 @@ public class StorageCleanJobHbaseUtil {
 
     public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException
{
         Configuration conf = HBaseConfiguration.create();
-        CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+        CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
         // get all kylin hbase tables
         try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
-            String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+            String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
             HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix +
".*");
             List<String> allTablesNeedToBeDropped = new ArrayList<String>();
             for (HTableDescriptor desc : tableDescriptors) {
                 String host = desc.getValue(IRealizationConstants.HTableTag);
-                if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
{
+                if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
                     //only take care htables that belongs to self, and created more than
2 days
                     allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
                 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 940d64a..1bafeb2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -57,9 +57,10 @@ public class HBaseClean extends AbstractApplication {
     private void cleanUp() {
         try {
             // get all kylin hbase tables
-            Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+            KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+            Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
             Admin hbaseAdmin = conn.getAdmin();
-            String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+            String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
             HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix +
".*");
             List<String> allTablesNeedToBeDropped = Lists.newArrayList();
             for (HTableDescriptor desc : tableDescriptors) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index a2f60d4..bb59242 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -43,9 +43,10 @@ public class HBaseUsage {
         Map<String, List<String>> envs = Maps.newHashMap();
 
         // get all kylin hbase tables
-        Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+        Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
         Admin hbaseAdmin = conn.getAdmin();
-        String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+        String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
         HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
         for (HTableDescriptor desc : tableDescriptors) {
             String host = desc.getValue(IRealizationConstants.HTableTag);

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index df4e912..0c35e46 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -56,10 +56,11 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
     Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 
     private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-        Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+        Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
         // get all kylin hbase tables
         Admin hbaseAdmin = conn.getAdmin();
-        String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+        String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
         HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
         List<String> allTablesNeedToBeDropped = new ArrayList<String>();
         for (HTableDescriptor desc : tableDescriptors) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f6b65ab..2ae21b4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -78,16 +78,17 @@ public class StorageCleanupJob extends AbstractApplication {
     protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
 
     private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-        CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+        CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
         // get all kylin hbase tables
-        Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+        Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
         Admin hbaseAdmin = conn.getAdmin();
-        String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+        String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
         HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
         List<String> allTablesNeedToBeDropped = new ArrayList<String>();
         for (HTableDescriptor desc : tableDescriptors) {
             String host = desc.getValue(IRealizationConstants.HTableTag);
-            if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
{
+            if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
                 //only take care htables that belongs to self, and created more than 2 days
                 allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
             }


Mime
View raw message