kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [01/13] kylin git commit: KYLIN-2417 Compare the performance between HDFSMetaStore and HBaseMetaStore [Forced Update!]
Date Wed, 22 Feb 2017 02:31:35 GMT
Repository: kylin
Updated Branches:
  refs/heads/master-hbase0.98 f5432f214 -> da9b080f9 (forced update)


KYLIN-2417 Compare the performance between HDFSMetaStore and HBaseMetaStore

Signed-off-by: Li Yang <liyang@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/65acd70e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/65acd70e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/65acd70e

Branch: refs/heads/master-hbase0.98
Commit: 65acd70ed4ff1a6f16a1ede95c1c9b85dc2f5578
Parents: 5d2e6c1
Author: xiefan46 <958034172@qq.com>
Authored: Wed Jan 25 17:07:43 2017 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Tue Feb 21 10:04:02 2017 +0800

----------------------------------------------------------------------
 .../common/persistence/ResourceStoreTest.java   | 42 +++++++++++++++++++-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 19 +++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/65acd70e/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index ddaf481..79a0c30 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -30,17 +30,34 @@ import java.util.List;
 import java.util.NavigableSet;
 
 import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
- * Be called by LocalFileResourceStoreTest and ITHBaseResourceStoreTest.
+ * Be called by LocalFileResourceStoreTest, ITHBaseResourceStoreTest and ITHDFSResourceStoreTest.
  */
 public class ResourceStoreTest {
 
+    private static final Logger logger = LoggerFactory.getLogger(ResourceStoreTest.class);
+
+    private static final String PERFORMANCE_TEST_ROOT_PATH = "/performance";
+
+    private static final int TEST_RESOURCE_COUNT = 1000;
+
     public static void testAStore(ResourceStore store) throws IOException {
         testBasics(store);
         testGetAllResources(store);
     }
 
+    public static void testPerformance(ResourceStore store) throws IOException {
+        logger.info("Test basic functions");
+        testAStore(store);
+        logger.info("Basic function ok. Start to test performance for class : " + store.getClass());
+        logger.info("Write metadata time : " + testWritePerformance(store));
+        logger.info("Read metadata time  " + testReadPerformance(store));
+        logger.info("Performance test end. Class : " + store.getClass());
+    }
+
     private static void testGetAllResources(ResourceStore store) throws IOException {
         final String folder = "/testFolder";
         List<StringEntity> result;
@@ -144,6 +161,29 @@ public class ResourceStoreTest {
         assertTrue(list == null || list.contains(path2) == false);
     }
 
+    private static long testWritePerformance(ResourceStore store) throws IOException {
+        store.deleteResource(PERFORMANCE_TEST_ROOT_PATH);
+        StringEntity content = new StringEntity("something");
+        long startTime = System.currentTimeMillis();
+        for (int i = 0; i < TEST_RESOURCE_COUNT; i++) {
+            String resourcePath = PERFORMANCE_TEST_ROOT_PATH + "/res_" + i;
+            store.putResource(resourcePath, content, 0, StringEntity.serializer);
+        }
+        return System.currentTimeMillis() - startTime;
+    }
+
+    private static long testReadPerformance(ResourceStore store) throws IOException {
+        long startTime = System.currentTimeMillis();
+        int step = 0; //avoid compiler optimization
+        for (int i = 0; i < TEST_RESOURCE_COUNT; i++) {
+            String resourcePath = PERFORMANCE_TEST_ROOT_PATH + "/res_" + i;
+            StringEntity t = store.getResource(resourcePath, StringEntity.class, StringEntity.serializer);
+            step |= t.toString().length();
+        }
+        logger.info("step : " + step);
+        return System.currentTimeMillis() - startTime;
+    }
+
     @SuppressWarnings("serial")
     public static class StringEntity extends RootPersistentEntity {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/65acd70e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index aa5a104..534839f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -27,8 +27,10 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.storage.hbase.HBaseResourceStore;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
@@ -84,4 +86,21 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
         assertTrue(fs.exists(new Path(path)));
     }
 
+    @Ignore
+    @Test
+    public void performanceTest() throws Exception{
+        //test hdfs performance
+        String oldUrl = kylinConfig.getMetadataUrl();
+        kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hdfs");
+        HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
+        ResourceStoreTest.testPerformance(store);
+        kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+
+        //test hbase
+        oldUrl = kylinConfig.getMetadataUrl();
+        kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hbase");
+        HBaseResourceStore store2 = new HBaseResourceStore(kylinConfig);
+        ResourceStoreTest.testPerformance(store2);
+        kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+    }
 }


Mime
View raw message