kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [27/39] kylin git commit: Add HDFSResourceStore to ResourceStore
Date Sat, 04 Feb 2017 06:16:49 GMT
Add HDFSResourceStore to ResourceStore

Signed-off-by: shaofengshi <shaofengshi@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f

Branch: refs/heads/spark-it
Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b
Parents: d3276e2
Author: xiefan46 <958034172@qq.com>
Authored: Mon Jan 23 10:51:12 2017 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800

----------------------------------------------------------------------
 .../kylin/common/persistence/ResourceStore.java       |  5 +++++
 .../kylin/storage/hdfs/ITHDFSResourceStoreTest.java   | 13 ++++---------
 .../apache/kylin/storage/hdfs/ITLockManagerTest.java  | 14 +++++++++-----
 .../kylin/storage/hbase/HBaseResourceStore.java       |  3 ++-
 .../apache/kylin/storage/hdfs/HDFSResourceStore.java  |  4 +++-
 5 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 9549569..25a0801 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -75,6 +75,11 @@ abstract public class ResourceStore {
             } catch (Throwable e) {
                 logger.warn("Failed to load HBaseResourceStore impl class: " + e.toString());
             }
+            try {
+                knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore",
ResourceStore.class));
+            } catch (Throwable e) {
+                logger.warn("Failed to load HDFSResourceStore impl class: " + e.toString());
+            }
         }
         return knownImpl;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 27d8a3c..47afb01 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -19,17 +19,12 @@
 package org.apache.kylin.storage.hdfs;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-
-/**
- * Created by xiefan on 17-1-10.
- */
 public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
 
     KylinConfig kylinConfig;
@@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
         this.cleanupTestMetadata();
     }
 
-
     @Test
     public void testResourceStoreBasic() throws Exception {
-        KylinConfig config = KylinConfig.getInstanceFromEnv();
-        ResourceStore store = new HDFSResourceStore(config);
+        String oldUrl = kylinConfig.getMetadataUrl();
+        kylinConfig.setProperty("kylin.metadata.url", "kylin_default_instance@hdfs");
+        HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
         ResourceStoreTest.testAStore(store);
+        kylinConfig.setProperty("kylin.metadata.url", oldUrl);
     }
 
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 2b58d30..a6cf43b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -58,6 +60,8 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
 
     private static final int REPETITIONS = QTY * 10;
 
+    private static final Logger logger = LoggerFactory.getLogger(ITLockManagerTest.class);
+
     @Before
     public void setup() throws Exception {
         this.createTestMetadata();
@@ -66,7 +70,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
         zkClient = CuratorFrameworkFactory.newClient(zkConnection, retryPolicy);
         zkClient.start();
         manager = new LockManager(kylinConfig, lockRootPath);
-        System.out.println("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
+        logger.info("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
 
     }
 
@@ -75,7 +79,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
         this.cleanupTestMetadata();
         zkClient.delete().deletingChildrenIfNeeded().forPath(lockRootPath);
         List<String> nodes = zkClient.getChildren().forPath("/");
-        System.out.println("nodes in zk after delete : " + nodes);
+        logger.info("nodes in zk after delete : " + nodes);
         manager.close();
     }
 
@@ -85,7 +89,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
         ResourceLock lock = manager.getLock("/dictionary/numberdict.json");
         lock.acquire();
         manager.releaseLock(lock);
-        System.out.println(zkClient.getChildren().forPath(lockRootPath + "/dictionary"));
+        logger.info(zkClient.getChildren().forPath(lockRootPath + "/dictionary").toString());
         List<String> nodes = zkClient.getChildren().forPath(lockRootPath + "/dictionary");
         assertEquals(1, nodes.size());
         assertEquals("numberdict.json", nodes.get(0));
@@ -193,10 +197,10 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
                 throw new IllegalStateException(clientName + " could not acquire the lock");
             }
             try {
-                System.out.println(clientName + " has the lock");
+                logger.info(clientName + " has the lock");
                 resource.use();
             } finally {
-                System.out.println(clientName + " releasing the lock");
+                logger.info(clientName + " releasing the lock");
                 lock.release(); // always release the lock in a finally block
             }
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 170e351..84670dd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -82,7 +82,8 @@ public class HBaseResourceStore extends ResourceStore {
         int cut = metadataUrl.indexOf('@');
         tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
         hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
+        if (!hbaseUrl.equals("hbase"))
+            throw new IOException("Can not create HBaseResourceStore. Url not match. Url:"
+ hbaseUrl);
         createHTableIfNeeded(getAllInOneTableName());
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index a746a97..bdd3917 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -57,12 +57,14 @@ public class HDFSResourceStore extends ResourceStore {
 
     private LockManager lockManager;
 
-    //public for test. Normal should be protected
     public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
         super(kylinConfig);
         String metadataUrl = kylinConfig.getMetadataUrl();
         int cut = metadataUrl.indexOf('@');
         String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0,
cut);
+        String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+        if(!hdfsUrl.equals("hdfs"))
+            throw new IOException("Can not create HDFSResourceStore. Url not match. Url:"
+ hdfsUrl);
         metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
         logger.info("meta dir name :" + metaDirName);
         createMetaFolder(metaDirName, kylinConfig);


Mime
View raw message