carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [11/50] incubator-carbondata git commit: [CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor lru cache size.
Date Tue, 18 Apr 2017 12:03:47 GMT
[CARBONDATA-890] For Spark 2.1 LRU cache size at driver is getting configured with the executor
lru cache size.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/99582a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/99582a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/99582a76

Branch: refs/heads/branch-1.1
Commit: 99582a762871c8a2c6cc9f5db43718d21e13de0c
Parents: 3966f99
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Tue Apr 11 09:59:45 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Wed Apr 12 14:10:34 2017 +0530

----------------------------------------------------------------------
 .../core/cache/CacheProviderTest.java           | 53 ++++++++++++++++++++
 .../scala/org/apache/spark/sql/CarbonEnv.scala  |  1 +
 2 files changed, 54 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/99582a76/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
index 1be5cb2..4c12259 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
@@ -17,16 +17,26 @@
 
 package org.apache.carbondata.core.cache;
 
+import java.io.IOException;
+import java.lang.reflect.Field;
+
 import org.apache.carbondata.core.cache.dictionary.Dictionary;
 import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
 import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
 import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.BlockIndexStore;
+import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
+import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
 import org.apache.carbondata.core.util.CarbonProperties;
 
+import org.apache.avro.Schema;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -39,6 +49,9 @@ public class CacheProviderTest {
     // enable lru cache by setting cache size
     CarbonProperties.getInstance()
         .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
+    // enable lru cache by setting cache size
+    CarbonProperties.getInstance()
+        .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "20");
   }
 
   @Test public void getInstance() throws Exception {
@@ -62,4 +75,44 @@ public class CacheProviderTest {
     assertTrue(reverseDictionaryCache instanceof ReverseDictionaryCache);
     assertFalse(reverseDictionaryCache instanceof ForwardDictionaryCache);
   }
+
+  /**
+   * to test the driver and executor lru memory configuration
+   *
+   * @throws IOException
+   * @throws NoSuchFieldException
+   * @throws IllegalAccessException
+   */
+  @Test public void driverExecutorCacheConfTest()
+      throws IOException, NoSuchFieldException, IllegalAccessException {
+    // get cache provider instance
+    CacheProvider cacheProvider = CacheProvider.getInstance();
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE,
"true");
+    Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache =
+        cacheProvider.createCache(CacheType.DRIVER_BTREE, "carbonStore");
+    Field carbonLRUCacheField = SegmentTaskIndexStore.class.getDeclaredField("lruCache");
+    carbonLRUCacheField.setAccessible(true);
+    CarbonLRUCache carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(driverCache);
+    Field lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
+    lruCacheMemorySizeField.setAccessible(true);
+    long lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
+    String driverCacheSize = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE);
+    assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), lruCacheMemorySize);
+    // drop cache
+    cacheProvider.dropAllCache();
+    // validation test for the executor memory.
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE,
"false");
+    Cache<TableBlockUniqueIdentifier, BlockIndexStore> executorCache =
+        cacheProvider.createCache(CacheType.EXECUTOR_BTREE, "carbonStore");
+    carbonLRUCacheField = BlockIndexStore.class.getSuperclass().getDeclaredField("lruCache");
+    carbonLRUCacheField.setAccessible(true);
+    carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(executorCache);
+    lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
+    lruCacheMemorySizeField.setAccessible(true);
+    lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
+    String executorCacheSize = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE);
+    assertEquals(1024 * 1024 * Integer.parseInt(executorCacheSize), lruCacheMemorySize);
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/99582a76/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 976a1b1..d426348 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -49,6 +49,7 @@ object CarbonEnv {
         LOGGER.info(s"carbon env initial: $storePath")
         new CarbonMetastore(sparkSession.conf, storePath)
       }
+      CarbonProperties.getInstance.addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE,
"true")
       carbonEnv = CarbonEnv(catalog)
       initialized = true
     }


Mime
View raw message