Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id DAD8A200C1F for ; Sat, 4 Feb 2017 07:16:27 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D9911160B6F; Sat, 4 Feb 2017 06:16:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 769CB160B71 for ; Sat, 4 Feb 2017 07:16:26 +0100 (CET) Received: (qmail 23859 invoked by uid 500); 4 Feb 2017 06:16:25 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 22553 invoked by uid 99); 4 Feb 2017 06:16:24 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Feb 2017 06:16:24 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0871EDFC47; Sat, 4 Feb 2017 06:16:24 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shaofengshi@apache.org To: commits@kylin.apache.org Date: Sat, 04 Feb 2017 06:16:49 -0000 Message-Id: <9f20500057b84ebd9c9d834587da9d88@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [27/39] kylin git commit: Add HDFSResourceStore to ResourceStore archived-at: Sat, 04 Feb 2017 06:16:28 -0000 Add HDFSResourceStore to ResourceStore Signed-off-by: shaofengshi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f Branch: refs/heads/spark-it Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b Parents: d3276e2 Author: xiefan46 <958034172@qq.com> Authored: Mon Jan 23 10:51:12 2017 +0800 Committer: shaofengshi Committed: Mon Jan 23 16:23:56 2017 +0800 ---------------------------------------------------------------------- .../kylin/common/persistence/ResourceStore.java | 5 +++++ .../kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 13 ++++--------- .../apache/kylin/storage/hdfs/ITLockManagerTest.java | 14 +++++++++----- .../kylin/storage/hbase/HBaseResourceStore.java | 3 ++- .../apache/kylin/storage/hdfs/HDFSResourceStore.java | 4 +++- 5 files changed, 23 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 9549569..25a0801 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -75,6 +75,11 @@ abstract public class ResourceStore { } catch (Throwable e) { logger.warn("Failed to load HBaseResourceStore impl class: " + e.toString()); } + try { + knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore", ResourceStore.class)); + } catch (Throwable e) { + logger.warn("Failed to load HDFSResourceStore impl class: " + e.toString()); + } } return knownImpl; } http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java index 27d8a3c..47afb01 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java @@ -19,17 +19,12 @@ package org.apache.kylin.storage.hdfs; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.ResourceStoreTest; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.junit.After; import org.junit.Before; import org.junit.Test; - -/** - * Created by xiefan on 17-1-10. - */ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase { KylinConfig kylinConfig; @@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase { this.cleanupTestMetadata(); } - @Test public void testResourceStoreBasic() throws Exception { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - ResourceStore store = new HDFSResourceStore(config); + String oldUrl = kylinConfig.getMetadataUrl(); + kylinConfig.setProperty("kylin.metadata.url", "kylin_default_instance@hdfs"); + HDFSResourceStore store = new HDFSResourceStore(kylinConfig); ResourceStoreTest.testAStore(store); + kylinConfig.setProperty("kylin.metadata.url", oldUrl); } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java index 2b58d30..a6cf43b 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java @@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.IOException; @@ -58,6 +60,8 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { private static final int REPETITIONS = QTY * 10; + private static final Logger logger = LoggerFactory.getLogger(ITLockManagerTest.class); + @Before public void setup() throws Exception { this.createTestMetadata(); @@ -66,7 +70,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { zkClient = CuratorFrameworkFactory.newClient(zkConnection, retryPolicy); zkClient.start(); manager = new LockManager(kylinConfig, lockRootPath); - System.out.println("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath)); + logger.info("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath)); } @@ -75,7 +79,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { this.cleanupTestMetadata(); zkClient.delete().deletingChildrenIfNeeded().forPath(lockRootPath); List nodes = zkClient.getChildren().forPath("/"); - System.out.println("nodes in zk after delete : " + nodes); + logger.info("nodes in zk after delete : " + nodes); manager.close(); } @@ -85,7 +89,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { ResourceLock lock = manager.getLock("/dictionary/numberdict.json"); lock.acquire(); manager.releaseLock(lock); - System.out.println(zkClient.getChildren().forPath(lockRootPath + "/dictionary")); + logger.info(zkClient.getChildren().forPath(lockRootPath + "/dictionary").toString()); List nodes = zkClient.getChildren().forPath(lockRootPath + "/dictionary"); assertEquals(1, nodes.size()); assertEquals("numberdict.json", nodes.get(0)); @@ -193,10 +197,10 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { throw new IllegalStateException(clientName + " could not acquire the lock"); } try { - System.out.println(clientName + " has the lock"); + logger.info(clientName + " has the lock"); resource.use(); } finally { - System.out.println(clientName + " releasing the lock"); + logger.info(clientName + " releasing the lock"); lock.release(); // always release the lock in a finally block } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index 170e351..84670dd 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -82,7 +82,8 @@ public class HBaseResourceStore extends ResourceStore { int cut = metadataUrl.indexOf('@'); tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut); hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); - + if (!hbaseUrl.equals("hbase")) + throw new IOException("Can not create HBaseResourceStore. Url not match. Url:" + hbaseUrl); createHTableIfNeeded(getAllInOneTableName()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java index a746a97..bdd3917 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java @@ -57,12 +57,14 @@ public class HDFSResourceStore extends ResourceStore { private LockManager lockManager; - //public for test. Normal should be protected public HDFSResourceStore(KylinConfig kylinConfig) throws Exception { super(kylinConfig); String metadataUrl = kylinConfig.getMetadataUrl(); int cut = metadataUrl.indexOf('@'); String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut); + String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); + if(!hdfsUrl.equals("hdfs")) + throw new IOException("Can not create HDFSResourceStore. Url not match. Url:" + hdfsUrl); metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME; logger.info("meta dir name :" + metaDirName); createMetaFolder(metaDirName, kylinConfig);