incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Added auto configuration of cache based on the maxdirectmemory setting. If the slab count is left to the default of -1 the maxdirectmemory value is used to the number of slabs to be allocated. So if the setting is -XX:MaxDirectMemory=
Date Fri, 08 Mar 2013 02:53:45 GMT
Updated Branches:
  refs/heads/0.2-dev 6cb3402f2 -> de4fece0b


Added auto configuration of cache based on the maxdirectmemory setting.  If the slab count
is left to the default of -1 the maxdirectmemory value is used to the number of slabs to be
allocated.  So if the setting is -XX:MaxDirectMemory=1100m then the number slabs would be
set automatically on startup to 8 (8 128MB slabs of block cache memory).  NOTE: Whatever the
value set is to, a default of 64MB is leftover to allow for any other JVM direct memory needs.
 This is the default amount the JVM allows when max direct memory is not set.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/de4fece0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/de4fece0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/de4fece0

Branch: refs/heads/0.2-dev
Commit: de4fece0beea4496daf58d5ebf4a6a3faae8cfe7
Parents: b298c08
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Mar 7 21:47:16 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Mar 7 21:47:16 2013 -0500

----------------------------------------------------------------------
 .../indexserver/DistributedIndexServer.java        |    8 ++-
 .../org/apache/blur/thrift/ThriftBlurServer.java   |   43 ++++++++++++++-
 .../src/main/resources/blur-default.properties     |    2 +-
 3 files changed, 47 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/de4fece0/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index a3fa7cb..aa5d7a8 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -474,8 +474,12 @@ public class DistributedIndexServer extends AbstractIndexServer {
     directory.setLockFactory(lockFactory);
     Directory dir = new BlockDirectory(table + "_" + shard, directory, _cache);
 
-    // Directory dir = shardContext.getDirectory();
-    // dir.setLockFactory(lockFactory);
+//    URI uri = hdfsDirPath.toUri();
+//    File dirFile = new File(uri);
+//    FSDirectory dir = MMapDirectory.open(dirFile,lockFactory);
+    
+//    Directory dir = shardContext.getDirectory();
+//    dir.setLockFactory(lockFactory);
 
     BlurIndex index;
     if (tableDescriptor.isReadOnly()) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/de4fece0/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
index a54a6e9..13a37a9 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
@@ -37,6 +37,7 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -70,7 +71,6 @@ import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.thrift.protocol.TJSONProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
 import org.apache.thrift.server.TServlet;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.Code;
@@ -120,7 +120,8 @@ public class ThriftBlurServer extends AbstractThriftServer {
     // in a slab when using a block size of 8,192
     int numberOfBlocksPerSlab = 16384;
     int blockSize = BlockDirectory.BLOCK_SIZE;
-    int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1);
+    int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1);
+    slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize);
     Cache cache;
     Configuration config = new Configuration();
 
@@ -249,7 +250,7 @@ public class ThriftBlurServer extends AbstractThriftServer {
     int threadCount = configuration.getInt(BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT, 32);
 
     Processor<Iface> processor = new Processor<Iface>(iface);
-    
+
     if (context != null) {
       context.addServlet(new ServletHolder(new TServlet(processor, new TJSONProtocol.Factory())),
"/rpc");
     }
@@ -275,6 +276,42 @@ public class ThriftBlurServer extends AbstractThriftServer {
     return server;
   }
 
+  private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize)
{
+    if (slabCount < 1) {
+      // -XX:MaxDirectMemorySize
+
+      long slabSize = numberOfBlocksPerSlab * blockSize;
+
+      List<String> inputArguments = ManagementFactory.getRuntimeMXBean().getInputArguments();
+      for (String arg : inputArguments) {
+        if (arg.startsWith("-XX:MaxDirectMemorySize")) {
+          long maxDirectMemorySize = getMaxDirectMemorySize(arg);
+          maxDirectMemorySize -= 64 * 1024 * 1024;
+          return (int) (maxDirectMemorySize / slabSize);
+        }
+      }
+
+      System.out.println(inputArguments);
+
+      throw new RuntimeException();
+    }
+    return 0;
+  }
+
+  private static long getMaxDirectMemorySize(String arg) {
+    int index = arg.lastIndexOf('=');
+    return parseNumber(arg.substring(index + 1).toLowerCase().replace(" ", ""));
+  }
+
+  private static long parseNumber(String number) {
+    if (number.endsWith("m")) {
+      return Long.parseLong(number.substring(0, number.length() - 1)) * 1024 * 1024;
+    } else if (number.endsWith("g")) {
+      return Long.parseLong(number.substring(0, number.length() - 1)) * 1024 * 1024 * 1024;
+    }
+    throw new RuntimeException("Cannot parse [" + number + "]");
+  }
+
   private static BlurFilterCache getFilterCache(BlurConfiguration configuration) {
     String _blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS);
     if (_blurFilterCacheClass != null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/de4fece0/src/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/src/blur-util/src/main/resources/blur-default.properties b/src/blur-util/src/main/resources/blur-default.properties
index 9c40e54..c7a740d 100644
--- a/src/blur-util/src/main/resources/blur-default.properties
+++ b/src/blur-util/src/main/resources/blur-default.properties
@@ -36,7 +36,7 @@ blur.shard.cache.max.timetolive=60000
 blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache
 blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup
 blur.shard.blockcache.direct.memory.allocation=true
-blur.shard.blockcache.slab.count=1
+blur.shard.blockcache.slab.count=-1
 blur.shard.buffercache.1024=8192
 blur.shard.buffercache.8192=8192
 blur.shard.safemodedelay=30000


Mime
View raw message