kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject [2/2] incubator-kylin git commit: KYLIN-1147 use pooled size for each cache(representing each realization)
Date Mon, 16 Nov 2015 08:03:47 GMT
KYLIN-1147 use pooled size for each cache(representing each realization)


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b9a19110
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b9a19110
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b9a19110

Branch: refs/heads/2.x-staging
Commit: b9a19110b826415faddf69d76dc69c2e25e07705
Parents: 91de85c
Author: honma <honma@ebay.com>
Authored: Mon Nov 16 16:08:06 2015 +0800
Committer: honma <honma@ebay.com>
Committed: Mon Nov 16 16:08:06 2015 +0800

----------------------------------------------------------------------
 .../cache/AbstractCacheFledgedQuery.java        |  28 ++---
 server/src/main/resources/ehcache-test.xml      |   3 -
 server/src/main/resources/ehcache.xml           |   3 -
 .../org/apache/kylin/rest/StorageCacheTest.java | 116 +++++++++++++++++++
 4 files changed, 131 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
index a7bdae0..745ddd1 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
@@ -5,7 +5,6 @@ import net.sf.ehcache.CacheManager;
 import net.sf.ehcache.Element;
 import net.sf.ehcache.config.CacheConfiguration;
 import net.sf.ehcache.config.Configuration;
-import net.sf.ehcache.config.MemoryUnit;
 import net.sf.ehcache.config.PersistenceConfiguration;
 import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
 
@@ -37,6 +36,14 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery,
TeeTup
         CACHE_MANAGER = cacheManager;
     }
 
+    /**
+     * This method is only useful non-spring injected test cases.
+     * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected.
+     * and the configuration for cache lies in server/src/main/resources/ehcache.xml
+     * 
+     * the cache named "StorageCache" acts like a template for each realization to
+     * create its own cache.
+     */
     private static void initCacheManger() {
         Configuration conf = new Configuration();
         conf.setMaxBytesLocalHeap("128M");
@@ -48,7 +55,7 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery,
TeeTup
                 eternal(false).//
                 timeToIdleSeconds(86400).//
                 diskExpiryThreadIntervalSeconds(0).//
-                maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
+                //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
                 persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
 
         CACHE_MANAGER.addCache(storageCache);
@@ -84,25 +91,20 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery,
TeeTup
         }
 
         if (CACHE_MANAGER.getCache(storageUUID) == null) {
-            logger.info("Cache for {} initting...", storageUUID);
+            logger.info("Cache for {} initializing...", storageUUID);
 
             //Create a Cache specifying its configuration.
             CacheConfiguration templateConf = CACHE_MANAGER.getCache(storageCacheTemplate).getCacheConfiguration();
-            PersistenceConfiguration pconf = templateConf.getPersistenceConfiguration();
-            if (pconf != null) {
-                logger.info("PersistenceConfiguration strategy: " + pconf.getStrategy());
-            } else {
-                logger.warn("PersistenceConfiguration is null");
-            }
-
-            Cache storageCache = new Cache(new CacheConfiguration(storageUUID, (int) templateConf.getMaxEntriesLocalHeap()).//
+
+            Cache storageCache = new Cache(new CacheConfiguration(storageUUID, 0).//
                     memoryStoreEvictionPolicy(templateConf.getMemoryStoreEvictionPolicy()).//
                     eternal(templateConf.isEternal()).//
                     timeToIdleSeconds(templateConf.getTimeToIdleSeconds()).//
-                    maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).persistence(pconf));
-            //TODO: deal with failed queries, and only cache too long query
+                    //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).//using
pooled size
+                    persistence(templateConf.getPersistenceConfiguration()));
 
             CACHE_MANAGER.addCache(storageCache);
+
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache-test.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache-test.xml b/server/src/main/resources/ehcache-test.xml
index 8ee2246..7cae6ce 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -3,7 +3,6 @@
            eternal="false"
            timeToIdleSeconds="86400"
            memoryStoreEvictionPolicy="LRU"
-           maxBytesLocalHeap="1M"
             >
         <persistence strategy="none"/>
     </cache>
@@ -11,7 +10,6 @@
            eternal="false"
            timeToIdleSeconds="86400"
            memoryStoreEvictionPolicy="LRU"
-           maxBytesLocalHeap="1M"
             >
         <persistence strategy="none"/>
     </cache>
@@ -19,7 +17,6 @@
            eternal="false"
            timeToLiveSeconds="10800"
            memoryStoreEvictionPolicy="LRU"
-           maxBytesLocalHeap="1M"
             >
         <persistence strategy="none"/>
     </cache>

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml
index 29fce05..3f39702 100644
--- a/server/src/main/resources/ehcache.xml
+++ b/server/src/main/resources/ehcache.xml
@@ -3,7 +3,6 @@
            eternal="false"
            timeToIdleSeconds="86400"
            memoryStoreEvictionPolicy="LRU"
-           maxBytesLocalHeap="50M"
             >
         <persistence strategy="none"/>
     </cache>
@@ -11,7 +10,6 @@
            eternal="false"
            timeToIdleSeconds="86400"
            memoryStoreEvictionPolicy="LRU"
-           maxBytesLocalHeap="50M"
             >
         <persistence strategy="none"/>
     </cache>
@@ -19,7 +17,6 @@
            eternal="false"
            timeToLiveSeconds="10800"
            memoryStoreEvictionPolicy="LRU"
-           maxBytesLocalHeap="50M"
             >
         <persistence strategy="none"/>
     </cache>

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
new file mode 100644
index 0000000..e781e00
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest;
+
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.apache.kylin.storage.ICachableStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery;
+import org.apache.kylin.storage.cache.CacheFledgedStaticQuery;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.google.common.collect.Range;
+
+public class StorageCacheTest extends ServiceTestBase {
+
+    public class MockedCachableStorageQuery implements ICachableStorageQuery {
+        private String uuid;
+
+        public MockedCachableStorageQuery(String uuid) {
+            this.uuid = uuid;
+        }
+
+        @Override
+        public boolean isDynamic() {
+            return false;
+        }
+
+        @Override
+        public Range<Long> getVolatilePeriod() {
+            throw new NotImplementedException();
+        }
+
+        @Override
+        public String getStorageUUID() {
+            return this.uuid;
+        }
+
+        @Override
+        public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo
returnTupleInfo) {
+            throw new NotImplementedException();
+        }
+    }
+
+    @Autowired
+    private CacheManager cacheManager;
+
+    @Before
+    public void setup() throws Exception {
+        AbstractCacheFledgedQuery.setCacheManager(cacheManager);
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("1"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("2"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("3"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("4"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("5"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("6"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("7"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("8"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("9"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("10"));
+        new CacheFledgedStaticQuery(new MockedCachableStorageQuery("11"));
+    }
+
+    @Test
+    public void test1() {
+        int oneM = 1 << 20;
+        cacheManager.getCache("1").put(new Element("xx", new byte[oneM]));
+        Element xx = cacheManager.getCache("1").get("xx");
+        Assert.assertEquals(oneM, ((byte[]) xx.getObjectValue()).length);
+
+        cacheManager.getCache("2").put(new Element("yy", new byte[3 * oneM]));
+        Element yy = cacheManager.getCache("2").get("yy");
+        Assert.assertEquals(3 * oneM, ((byte[]) yy.getObjectValue()).length);
+
+        cacheManager.getCache("3").put(new Element("zz", new byte[10 * oneM]));
+        Element zz = cacheManager.getCache("3").get("zz");
+        Assert.assertEquals(null, zz);
+
+        cacheManager.getCache("4").put(new Element("aa", new byte[oneM]));
+        Element aa = cacheManager.getCache("4").get("aa");
+        Assert.assertEquals(oneM, ((byte[]) aa.getObjectValue()).length);
+
+        cacheManager.getCache("2").put(new Element("bb", new byte[3 * oneM]));
+        Element bb = cacheManager.getCache("2").get("bb");
+        Assert.assertEquals(3 * oneM, ((byte[]) bb.getObjectValue()).length);
+    }
+
+    @Test
+    public void test2() {
+    }
+}


Mime
View raw message