Return-Path: X-Original-To: apmail-kylin-commits-archive@minotaur.apache.org Delivered-To: apmail-kylin-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9221718DED for ; Mon, 16 Nov 2015 08:03:51 +0000 (UTC) Received: (qmail 42250 invoked by uid 500); 16 Nov 2015 08:03:51 -0000 Delivered-To: apmail-kylin-commits-archive@kylin.apache.org Received: (qmail 42222 invoked by uid 500); 16 Nov 2015 08:03:51 -0000 Mailing-List: contact commits-help@kylin.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.incubator.apache.org Delivered-To: mailing list commits@kylin.incubator.apache.org Received: (qmail 42213 invoked by uid 99); 16 Nov 2015 08:03:51 -0000 Received: from Unknown (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 16 Nov 2015 08:03:51 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 1514C1A0C02 for ; Mon, 16 Nov 2015 08:03:51 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.77 X-Spam-Level: * X-Spam-Status: No, score=1.77 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01] autolearn=disabled Received: from mx1-eu-west.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id dzy2kbgL3WMX for ; Mon, 16 Nov 2015 08:03:48 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id 442E62026E for ; Mon, 16 Nov 2015 08:03:47 +0000 (UTC) Received: (qmail 42142 invoked by uid 99); 16 Nov 2015 08:03:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 16 Nov 2015 08:03:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 349E6DFCC9; Mon, 16 Nov 2015 08:03:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mahongbin@apache.org To: commits@kylin.incubator.apache.org Date: Mon, 16 Nov 2015 08:03:47 -0000 Message-Id: In-Reply-To: <3c6fd5b7a22d4674b96c1fc0020817f4@git.apache.org> References: <3c6fd5b7a22d4674b96c1fc0020817f4@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] incubator-kylin git commit: KYLIN-1147 use pooled size for each cache(representing each realization) KYLIN-1147 use pooled size for each cache(representing each realization) Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b9a19110 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b9a19110 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b9a19110 Branch: refs/heads/2.x-staging Commit: b9a19110b826415faddf69d76dc69c2e25e07705 Parents: 91de85c Author: honma Authored: Mon Nov 16 16:08:06 2015 +0800 Committer: honma Committed: Mon Nov 16 16:08:06 2015 +0800 ---------------------------------------------------------------------- .../cache/AbstractCacheFledgedQuery.java | 28 ++--- server/src/main/resources/ehcache-test.xml | 3 - server/src/main/resources/ehcache.xml | 3 - .../org/apache/kylin/rest/StorageCacheTest.java | 116 +++++++++++++++++++ 4 files changed, 131 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java index a7bdae0..745ddd1 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java @@ -5,7 +5,6 @@ import net.sf.ehcache.CacheManager; import net.sf.ehcache.Element; import net.sf.ehcache.config.CacheConfiguration; import net.sf.ehcache.config.Configuration; -import net.sf.ehcache.config.MemoryUnit; import net.sf.ehcache.config.PersistenceConfiguration; import net.sf.ehcache.store.MemoryStoreEvictionPolicy; @@ -37,6 +36,14 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup CACHE_MANAGER = cacheManager; } + /** + * This method is only useful non-spring injected test cases. + * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected. + * and the configuration for cache lies in server/src/main/resources/ehcache.xml + * + * the cache named "StorageCache" acts like a template for each realization to + * create its own cache. + */ private static void initCacheManger() { Configuration conf = new Configuration(); conf.setMaxBytesLocalHeap("128M"); @@ -48,7 +55,7 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup eternal(false).// timeToIdleSeconds(86400).// diskExpiryThreadIntervalSeconds(0).// - maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).// + //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).// persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE))); CACHE_MANAGER.addCache(storageCache); @@ -84,25 +91,20 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup } if (CACHE_MANAGER.getCache(storageUUID) == null) { - logger.info("Cache for {} initting...", storageUUID); + logger.info("Cache for {} initializing...", storageUUID); //Create a Cache specifying its configuration. CacheConfiguration templateConf = CACHE_MANAGER.getCache(storageCacheTemplate).getCacheConfiguration(); - PersistenceConfiguration pconf = templateConf.getPersistenceConfiguration(); - if (pconf != null) { - logger.info("PersistenceConfiguration strategy: " + pconf.getStrategy()); - } else { - logger.warn("PersistenceConfiguration is null"); - } - - Cache storageCache = new Cache(new CacheConfiguration(storageUUID, (int) templateConf.getMaxEntriesLocalHeap()).// + + Cache storageCache = new Cache(new CacheConfiguration(storageUUID, 0).// memoryStoreEvictionPolicy(templateConf.getMemoryStoreEvictionPolicy()).// eternal(templateConf.isEternal()).// timeToIdleSeconds(templateConf.getTimeToIdleSeconds()).// - maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).persistence(pconf)); - //TODO: deal with failed queries, and only cache too long query + //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).//using pooled size + persistence(templateConf.getPersistenceConfiguration())); CACHE_MANAGER.addCache(storageCache); + } } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache-test.xml ---------------------------------------------------------------------- diff --git a/server/src/main/resources/ehcache-test.xml b/server/src/main/resources/ehcache-test.xml index 8ee2246..7cae6ce 100644 --- a/server/src/main/resources/ehcache-test.xml +++ b/server/src/main/resources/ehcache-test.xml @@ -3,7 +3,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="1M" > @@ -11,7 +10,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="1M" > @@ -19,7 +17,6 @@ eternal="false" timeToLiveSeconds="10800" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="1M" > http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache.xml ---------------------------------------------------------------------- diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml index 29fce05..3f39702 100644 --- a/server/src/main/resources/ehcache.xml +++ b/server/src/main/resources/ehcache.xml @@ -3,7 +3,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="50M" > @@ -11,7 +10,6 @@ eternal="false" timeToIdleSeconds="86400" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="50M" > @@ -19,7 +17,6 @@ eternal="false" timeToLiveSeconds="10800" memoryStoreEvictionPolicy="LRU" - maxBytesLocalHeap="50M" > http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java new file mode 100644 index 0000000..e781e00 --- /dev/null +++ b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest; + +import net.sf.ehcache.CacheManager; +import net.sf.ehcache.Element; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.kylin.metadata.realization.SQLDigest; +import org.apache.kylin.metadata.tuple.ITupleIterator; +import org.apache.kylin.rest.service.ServiceTestBase; +import org.apache.kylin.storage.ICachableStorageQuery; +import org.apache.kylin.storage.StorageContext; +import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; +import org.apache.kylin.storage.cache.CacheFledgedStaticQuery; +import org.apache.kylin.storage.tuple.TupleInfo; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import com.google.common.collect.Range; + +public class StorageCacheTest extends ServiceTestBase { + + public class MockedCachableStorageQuery implements ICachableStorageQuery { + private String uuid; + + public MockedCachableStorageQuery(String uuid) { + this.uuid = uuid; + } + + @Override + public boolean isDynamic() { + return false; + } + + @Override + public Range getVolatilePeriod() { + throw new NotImplementedException(); + } + + @Override + public String getStorageUUID() { + return this.uuid; + } + + @Override + public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { + throw new NotImplementedException(); + } + } + + @Autowired + private CacheManager cacheManager; + + @Before + public void setup() throws Exception { + AbstractCacheFledgedQuery.setCacheManager(cacheManager); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("1")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("2")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("3")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("4")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("5")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("6")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("7")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("8")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("9")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("10")); + new CacheFledgedStaticQuery(new MockedCachableStorageQuery("11")); + } + + @Test + public void test1() { + int oneM = 1 << 20; + cacheManager.getCache("1").put(new Element("xx", new byte[oneM])); + Element xx = cacheManager.getCache("1").get("xx"); + Assert.assertEquals(oneM, ((byte[]) xx.getObjectValue()).length); + + cacheManager.getCache("2").put(new Element("yy", new byte[3 * oneM])); + Element yy = cacheManager.getCache("2").get("yy"); + Assert.assertEquals(3 * oneM, ((byte[]) yy.getObjectValue()).length); + + cacheManager.getCache("3").put(new Element("zz", new byte[10 * oneM])); + Element zz = cacheManager.getCache("3").get("zz"); + Assert.assertEquals(null, zz); + + cacheManager.getCache("4").put(new Element("aa", new byte[oneM])); + Element aa = cacheManager.getCache("4").get("aa"); + Assert.assertEquals(oneM, ((byte[]) aa.getObjectValue()).length); + + cacheManager.getCache("2").put(new Element("bb", new byte[3 * oneM])); + Element bb = cacheManager.getCache("2").get("bb"); + Assert.assertEquals(3 * oneM, ((byte[]) bb.getObjectValue()).length); + } + + @Test + public void test2() { + } +}