incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Refactoring to allow for easier configuration.
Date Thu, 10 Oct 2013 18:16:39 GMT
Updated Branches:
  refs/heads/master 28470cd67 -> dc94a2ca9


Refactoring to allow for easier configuration.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/dc94a2ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/dc94a2ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/dc94a2ca

Branch: refs/heads/master
Commit: dc94a2ca99f4de0f40e7825556f7c1745e0f64fb
Parents: 28470cd
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Oct 10 14:16:15 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Oct 10 14:16:32 2013 -0400

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurShardServer.java      | 67 +++-----------------
 .../store/BlockCacheDirectoryFactoryV1.java     | 58 ++++++++++++++++-
 .../store/BlockCacheDirectoryFactoryV2.java     | 12 +++-
 .../org/apache/blur/utils/BlurConstants.java    |  2 +-
 .../src/main/resources/blur-default.properties  | 11 ++--
 docs/cluster-setup.html                         | 17 ++---
 6 files changed, 88 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 82cf541..926c9e9 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -28,9 +28,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_MAX_HEAP_PER_ROW_FETCH;
 import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT;
-import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION;
-import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT;
-import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_VERSION;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FETCHCOUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
@@ -77,10 +75,6 @@ import org.apache.blur.server.TableContext;
 import org.apache.blur.store.BlockCacheDirectoryFactory;
 import org.apache.blur.store.BlockCacheDirectoryFactoryV1;
 import org.apache.blur.store.BlockCacheDirectoryFactoryV2;
-import org.apache.blur.store.blockcache.BlockCache;
-import org.apache.blur.store.blockcache.BlockDirectory;
-import org.apache.blur.store.blockcache.BlockDirectoryCache;
-import org.apache.blur.store.blockcache.Cache;
 import org.apache.blur.store.buffer.BufferStore;
 import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol;
 import org.apache.blur.thirdparty.thrift_0_9_0.server.TServlet;
@@ -157,48 +151,15 @@ public class ThriftBlurShardServer extends ThriftServer {
     BlockCacheDirectoryFactory blockCacheDirectoryFactory;
     // Alternate BlockCacheDirectoryFactory support currently disabled in 0.2.0,
     // look for it in 0.2.1
-    boolean experimentalBlockCache = configuration.getBoolean(BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE,
false);
-    experimentalBlockCache = false;
-    if (!experimentalBlockCache) {
-      // setup block cache
-      // 134,217,728 is the slab size, therefore there are 16,384 blocks
-      // in a slab when using a block size of 8,192
-      int numberOfBlocksPerSlab = 16384;
-      int blockSize = BlockDirectory.BLOCK_SIZE;
-      int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1);
-      slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize);
-      Cache cache;
-      if (slabCount >= 1) {
-        BlockCache blockCache;
-        boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION,
true);
-
-        int slabSize = numberOfBlocksPerSlab * blockSize;
-        LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set
to [{1}]", slabCount,
-            directAllocation);
-        LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}]
slabs and use ~[{2}] bytes",
-            slabSize, slabCount, ((long) slabCount * (long) slabSize));
-
-        try {
-          long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize;
-          blockCache = new BlockCache(directAllocation, totalMemory, slabSize);
-        } catch (OutOfMemoryError e) {
-          if ("Direct buffer memory".equals(e.getMessage())) {
-            System.err
-                .println("The max direct memory is too low.  Either increase by setting (-XX:MaxDirectMemorySize=<size>g
-XX:+UseLargePages) or disable direct allocation by (blur.shard.blockcache.direct.memory.allocation=false)
in blur-site.properties");
-            System.exit(1);
-          }
-          throw e;
-        }
-        cache = new BlockDirectoryCache(blockCache);
-      } else {
-        cache = BlockDirectory.NO_CACHE;
-      }
-      blockCacheDirectoryFactory = new BlockCacheDirectoryFactoryV1(cache);
-    } else {
-      long totalNumberOfBytes = VM.maxDirectMemory() - _64MB;
+    String blockCacheVersion = configuration.get(BLUR_SHARD_BLOCK_CACHE_VERSION, "v2");
+    long totalNumberOfBytes = VM.maxDirectMemory() - _64MB;
+    if (blockCacheVersion.equals("v1")) {
+      blockCacheDirectoryFactory = new BlockCacheDirectoryFactoryV1(configuration, totalNumberOfBytes);
+    } else if (blockCacheVersion.equals("v2")) {
       blockCacheDirectoryFactory = new BlockCacheDirectoryFactoryV2(configuration, totalNumberOfBytes);
+    } else {
+      throw new RuntimeException("Unknown block cache version [" + blockCacheVersion + "]
can be [v1,v2]");
     }
-
     LOG.info("Shard Server using index [{0}] bind address [{1}] random port assignment [{2}]",
serverIndex, bindAddress
         + ":" + bindPort, randomPort);
 
@@ -296,18 +257,6 @@ public class ThriftBlurShardServer extends ThriftServer {
     return server;
   }
 
-  private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize)
{
-    if (slabCount < 0) {
-      long slabSize = numberOfBlocksPerSlab * blockSize;
-      long maxDirectMemorySize = VM.maxDirectMemory() - _64MB;
-      if (maxDirectMemorySize < slabSize) {
-        throw new RuntimeException("Auto slab setup cannot happen, JVM option -XX:MaxDirectMemorySize
not set.");
-      }
-      return (int) (maxDirectMemorySize / slabSize);
-    }
-    return slabCount;
-  }
-
   private static BlurFilterCache getFilterCache(BlurConfiguration configuration) {
     String blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS);
     if (blurFilterCacheClass != null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
index 182b494..8e9aa11 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
@@ -16,18 +16,61 @@
  */
 package org.apache.blur.store;
 
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT;
+
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.store.blockcache.BlockCache;
 import org.apache.blur.store.blockcache.BlockDirectory;
+import org.apache.blur.store.blockcache.BlockDirectoryCache;
 import org.apache.blur.store.blockcache.Cache;
 import org.apache.lucene.store.Directory;
 
 public class BlockCacheDirectoryFactoryV1 implements BlockCacheDirectoryFactory {
 
+  private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV1.class);
+
   private final Cache _cache;
-  
-  public BlockCacheDirectoryFactoryV1(Cache cache) {
+
+  public BlockCacheDirectoryFactoryV1(BlurConfiguration configuration, long totalNumberOfBytes)
{
+    // setup block cache
+    // 134,217,728 is the slab size, therefore there are 16,384 blocks
+    // in a slab when using a block size of 8,192
+    int numberOfBlocksPerSlab = 16384;
+    int blockSize = BlockDirectory.BLOCK_SIZE;
+    int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1);
+    slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize, totalNumberOfBytes);
+    Cache cache;
+    if (slabCount >= 1) {
+      BlockCache blockCache;
+      boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION,
true);
+
+      int slabSize = numberOfBlocksPerSlab * blockSize;
+      LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to
[{1}]", slabCount,
+          directAllocation);
+      LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs
and use ~[{2}] bytes",
+          slabSize, slabCount, ((long) slabCount * (long) slabSize));
+
+      try {
+        long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize;
+        blockCache = new BlockCache(directAllocation, totalMemory, slabSize);
+      } catch (OutOfMemoryError e) {
+        if ("Direct buffer memory".equals(e.getMessage())) {
+          System.err
+              .println("The max direct memory is too low.  Either increase by setting (-XX:MaxDirectMemorySize=<size>g
-XX:+UseLargePages) or disable direct allocation by (blur.shard.blockcache.direct.memory.allocation=false)
in blur-site.properties");
+          System.exit(1);
+        }
+        throw e;
+      }
+      cache = new BlockDirectoryCache(blockCache);
+    } else {
+      cache = BlockDirectory.NO_CACHE;
+    }
     _cache = cache;
   }
 
@@ -36,4 +79,15 @@ public class BlockCacheDirectoryFactoryV1 implements BlockCacheDirectoryFactory
     return new BlockDirectory(name, directory, _cache, blockCacheFileTypes);
   }
 
+  private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize,
long totalNumberOfBytes) {
+    if (slabCount < 0) {
+      long slabSize = numberOfBlocksPerSlab * blockSize;
+      if (totalNumberOfBytes < slabSize) {
+        throw new RuntimeException("Auto slab setup cannot happen, JVM option -XX:MaxDirectMemorySize
not set.");
+      }
+      return (int) (totalNumberOfBytes / slabSize);
+    }
+    return slabCount;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
index e3a7007..d61dd73 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
@@ -33,13 +33,17 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory
   private Cache _cache;
 
   public BlockCacheDirectoryFactoryV2(BlurConfiguration configuration, long totalNumberOfBytes)
{
-    int fileBufferSize = 8192;
+    final int fileBufferSize = 8192;
+    final int blockSize = 8192;
+    final STORE store = STORE.OFF_HEAP;
+    
     FileNameBlockSize fileNameBlockSize = new FileNameBlockSize() {
       @Override
       public int getBlockSize(String directoryName, String fileName) {
-        return 8192;
+        return blockSize;
       }
     };
+    
     FileNameFilter readFilter = new FileNameFilter() {
       @Override
       public boolean accept(String directoryName, String fileName) {
@@ -49,6 +53,7 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory
         return true;
       }
     };
+    
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(String directoryName, String fileName) {
@@ -58,8 +63,9 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory
         return true;
       }
     };
+    
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, fileNameBlockSize, readFilter,
writeFilter,
-        STORE.OFF_HEAP);
+        store);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 1373ad2..993208b 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -44,7 +44,7 @@ public class BlurConstants {
   public static final String BLUR_SHARD_BIND_ADDRESS = "blur.shard.bind.address";
   public static final String BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION = "blur.shard.blockcache.direct.memory.allocation";
   public static final String BLUR_SHARD_BLOCKCACHE_SLAB_COUNT = "blur.shard.blockcache.slab.count";
-  public static final String BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE = "blur.shard.experimental.block.cache";
+  public static final String BLUR_SHARD_BLOCK_CACHE_VERSION = "blur.shard.block.cache.version";
   public static final String BLUR_SHARD_SAFEMODEDELAY = "blur.shard.safemodedelay";
   public static final String BLUR_CONTROLLER_HOSTNAME = "blur.controller.hostname";
   public static final String BLUR_CONTROLLER_BIND_PORT = "blur.controller.bind.port";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 48aaabf..922dd6d 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -72,14 +72,13 @@ blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurInd
 # Throttles the warmup to 30MB/s across all the warmup threads
 blur.shard.index.warmup.throttle=30000000
 
-# By default the block cache using off heap memory
-blur.shard.blockcache.direct.memory.allocation=true
+# By default the v2 version of the block cache is enabled
+blur.shard.block.cache.version=v2
 
-# By default the experimental block cache is off
-blur.shard.experimental.block.cache=false
+# v1 version of block cache only. By default the block cache using off heap memory
+blur.shard.blockcache.direct.memory.allocation=true
 
-# The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab
equals 128MB
-# The auto config is detected through the MaxDirectoryMemorySize provided to the JVM
+# v1 version of block cache only. The slabs in the blockcache are automatically configured
by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through the MaxDirectoryMemorySize
provided to the JVM
 blur.shard.blockcache.slab.count=-1
 
 # The number of 1K byte buffers

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc94a2ca/docs/cluster-setup.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html
index 1c02ca7..ad5eb37 100644
--- a/docs/cluster-setup.html
+++ b/docs/cluster-setup.html
@@ -302,16 +302,17 @@ blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurInd
 # Throttles the warmup to 30MB/s across all the warmup threads
 blur.shard.index.warmup.throttle=30000000
 
-# By default the block cache using off heap memory
-blur.shard.blockcache.direct.memory.allocation=true
+# By default the v2 version of the block cache is enabled
+blur.shard.block.cache.version=v2
 
-# By default the experimental block cache is off
-blur.shard.experimental.block.cache=false
+# v1 version of block cache only. By default the block cache using
+# off heap memory
+blur.shard.blockcache.direct.memory.allocation=true
 
-# The slabs in the blockcache are automatically configured by 
-# default (-1) otherwise 1 slab equals 128MB. The auto config
-# is detected through the MaxDirectoryMemorySize provided to 
-# the JVM
+# v1 version of block cache only. The slabs in the blockcache are 
+# automatically configured by default (-1) otherwise 1 slab equals 
+# 128MB.  The auto config is detected through the MaxDirectoryMemorySize 
+# provided to the JVM
 blur.shard.blockcache.slab.count=-1
 
 # The number of 1K byte buffers


Mime
View raw message