incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [17/47] git commit: Renamed occurences of bank to slab in BlockCache
Date Mon, 03 Sep 2012 00:32:19 GMT
Renamed occurences of bank to slab in BlockCache


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/172a058a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/172a058a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/172a058a

Branch: refs/heads/master
Commit: 172a058a9c4ff46205ce86a7a518a73fbbe08f5d
Parents: d4a4353
Author: Scott Leberknight <sleberkn@nearinfinity.com>
Authored: Mon Aug 27 18:04:46 2012 -0400
Committer: Scott Leberknight <sleberkn@nearinfinity.com>
Committed: Mon Aug 27 18:04:46 2012 -0400

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurShardServer.java         |   62 +++++++-------
 .../blur/store/blockcache/BlockCache.java          |   64 +++++++-------
 .../blur/store/blockcache/BlockCacheLocation.java  |   10 +-
 .../blur/store/BenchmarkDirectory.java             |    8 +-
 .../blur/store/BenchmarkDirectoryNrt.java          |    8 +-
 5 files changed, 76 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java
b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java
index bb1243d..6b1d07f 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/ThriftBlurShardServer.java
@@ -16,28 +16,6 @@
 
 package com.nearinfinity.blur.thrift;
 
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_CLUSTER_NAME;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_MAX_CLAUSE_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE;
-import static com.nearinfinity.blur.utils.BlurUtil.quietClose;
-
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
@@ -73,6 +51,28 @@ import com.nearinfinity.blur.thrift.generated.Blur.Iface;
 import com.nearinfinity.blur.utils.BlurUtil;
 import com.nearinfinity.blur.zookeeper.ZkUtils;
 
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_CLUSTER_NAME;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_MAX_CLAUSE_COUNT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE;
+import static com.nearinfinity.blur.utils.BlurUtil.quietClose;
+
 public class ThriftBlurShardServer extends ThriftServer {
 
   private static final Log LOG = LogFactory.getLog(ThriftBlurShardServer.class);
@@ -90,26 +90,26 @@ public class ThriftBlurShardServer extends ThriftServer {
 
   public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration)
throws Exception {
     // setup block cache
-    // 134,217,728 is the bank size, therefore there are 16,384 block
-    // in a bank when using a block of 8,192
-    int numberOfBlocksPerBank = 16384;
+    // 134,217,728 is the slab size, therefore there are 16,384 blocks
+    // in a slab when using a block size of 8,192
+    int numberOfBlocksPerSlab = 16384;
     int blockSize = BlockDirectory.BLOCK_SIZE;
-    int bankCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1);
+    int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1);
     Cache cache;
     Configuration config = new Configuration();
     BlurMetrics blurMetrics = new BlurMetrics(config);
-    if (bankCount >= 1) {
+    if (slabCount >= 1) {
       BlockCache blockCache;
       boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION,
true);
 
-      int slabSize = numberOfBlocksPerBank * blockSize;
-      LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to
[{1}]", bankCount, directAllocation);
-      LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs
and use ~[{2}] bytes", slabSize, bankCount, ((long) bankCount * (long) slabSize));
+      int slabSize = numberOfBlocksPerSlab * blockSize;
+      LOG.info("Number of slabs of block cache [{0}] with direct memory allocation set to
[{1}]", slabCount, directAllocation);
+      LOG.info("Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs
and use ~[{2}] bytes", slabSize, slabCount, ((long) slabCount * (long) slabSize));
 
       BufferStore.init(configuration, blurMetrics);
 
       try {
-        long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank * (long) blockSize;
+        long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize;
         blockCache = new BlockCache(blurMetrics, directAllocation, totalMemory, slabSize,
blockSize);
       } catch (OutOfMemoryError e) {
         if ("Direct buffer memory".equals(e.getMessage())) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
index 349fc34..ecd2960 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
@@ -13,11 +13,11 @@ public class BlockCache {
   public static final int _128M = 134217728;
   public static final int _32K = 32768;
   private final ConcurrentMap<BlockCacheKey, BlockCacheLocation> _cache;
-  private final ByteBuffer[] _banks;
+  private final ByteBuffer[] _slabs;
   private final BlockLocks[] _locks;
   private final AtomicInteger[] _lockCounters;
   private final int _blockSize;
-  private final int _numberOfBlocksPerBank;
+  private final int _numberOfBlocksPerSlab;
   private final int _maxEntries;
   private final BlurMetrics _metrics;
   
@@ -31,20 +31,20 @@ public class BlockCache {
   
   public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory, int
slabSize, int blockSize) {
     _metrics = metrics;
-    _numberOfBlocksPerBank = slabSize / blockSize;
-    int numberOfBanks = (int) (totalMemory / slabSize);
+    _numberOfBlocksPerSlab = slabSize / blockSize;
+    int numberOfSlabs = (int) (totalMemory / slabSize);
     
-    _banks = new ByteBuffer[numberOfBanks];
-    _locks = new BlockLocks[numberOfBanks];
-    _lockCounters = new AtomicInteger[numberOfBanks];
-    _maxEntries = (_numberOfBlocksPerBank * numberOfBanks) - 1;
-    for (int i = 0; i < numberOfBanks; i++) {
+    _slabs = new ByteBuffer[numberOfSlabs];
+    _locks = new BlockLocks[numberOfSlabs];
+    _lockCounters = new AtomicInteger[numberOfSlabs];
+    _maxEntries = (_numberOfBlocksPerSlab * numberOfSlabs) - 1;
+    for (int i = 0; i < numberOfSlabs; i++) {
       if (directAllocation) {
-        _banks[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerBank * blockSize);
+        _slabs[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerSlab * blockSize);
       } else {
-        _banks[i] = ByteBuffer.allocate(_numberOfBlocksPerBank * blockSize);
+        _slabs[i] = ByteBuffer.allocate(_numberOfBlocksPerSlab * blockSize);
       }
-      _locks[i] = new BlockLocks(_numberOfBlocksPerBank);
+      _locks[i] = new BlockLocks(_numberOfBlocksPerSlab);
       _lockCounters[i] = new AtomicInteger();
     }
 
@@ -62,11 +62,11 @@ public class BlockCache {
     if (location == null) {
       return;
     }
-    int bankId = location.getBankId();
+    int slabId = location.getSlabId();
     int block = location.getBlock();
     location.setRemoved(true);
-    _locks[bankId].clear(block);
-    _lockCounters[bankId].decrementAndGet();
+    _locks[slabId].clear(block);
+    _lockCounters[slabId].decrementAndGet();
     _metrics.blockCacheEviction.incrementAndGet();
     _metrics.blockCacheSize.decrementAndGet();
   }
@@ -85,11 +85,11 @@ public class BlockCache {
     if (location.isRemoved()) {
       return false;
     }
-    int bankId = location.getBankId();
+    int slabId = location.getSlabId();
     int offset = location.getBlock() * _blockSize;
-    ByteBuffer bank = getBank(bankId);
-    bank.position(offset);
-    bank.put(data, 0, _blockSize);
+    ByteBuffer slab = getSlab(slabId);
+    slab.position(offset);
+    slab.put(data, 0, _blockSize);
     if (newLocation) {
       releaseLocation(_cache.put(blockCacheKey.clone(), location));
       _metrics.blockCacheSize.incrementAndGet();
@@ -105,12 +105,12 @@ public class BlockCache {
     if (location.isRemoved()) {
       return false;
     }
-    int bankId = location.getBankId();
+    int slabId = location.getSlabId();
     int offset = location.getBlock() * _blockSize;
     location.touch();
-    ByteBuffer bank = getBank(bankId);
-    bank.position(offset + blockOffset);
-    bank.get(buffer, off, length);
+    ByteBuffer slab = getSlab(slabId);
+    slab.position(offset + blockOffset);
+    slab.get(buffer, off, length);
     return true;
   }
 
@@ -123,18 +123,18 @@ public class BlockCache {
     // This is a tight loop that will try and find a location to
     // place the block before giving up
     for (int j = 0; j < 10; j++) {
-      OUTER: for (int bankId = 0; bankId < _banks.length; bankId++) {
-        AtomicInteger bitSetCounter = _lockCounters[bankId];
-        BlockLocks bitSet = _locks[bankId];
-        if (bitSetCounter.get() == _numberOfBlocksPerBank) {
+      OUTER: for (int slabId = 0; slabId < _slabs.length; slabId++) {
+        AtomicInteger bitSetCounter = _lockCounters[slabId];
+        BlockLocks bitSet = _locks[slabId];
+        if (bitSetCounter.get() == _numberOfBlocksPerSlab) {
           // if bitset is full
           continue OUTER;
         }
         // this check needs to spin, if a lock was attempted but not obtained
-        // the rest of the bank should not be skipped
+        // the rest of the slab should not be skipped
         int bit = bitSet.nextClearBit(0);
         INNER: while (bit != -1) {
-          if (bit >= _numberOfBlocksPerBank) {
+          if (bit >= _numberOfBlocksPerSlab) {
             // bit set is full
             continue OUTER;
           }
@@ -146,7 +146,7 @@ public class BlockCache {
             continue INNER;
           } else {
             // lock obtained
-            location.setBankId(bankId);
+            location.setSlabId(slabId);
             location.setBlock(bit);
             bitSetCounter.incrementAndGet();
             return true;
@@ -163,8 +163,8 @@ public class BlockCache {
     }
   }
 
-  private ByteBuffer getBank(int bankId) {
-    return _banks[bankId].duplicate();
+  private ByteBuffer getSlab(int slabId) {
+    return _slabs[slabId].duplicate();
   }
 
   public int getSize() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
index 4940ce1..b394894 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
@@ -6,7 +6,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class BlockCacheLocation {
 
     private int _block;
-    private int _bankId;
+    private int _slabId;
     private long _lastAccess = System.currentTimeMillis();
     private long _accesses;
     private AtomicBoolean _removed = new AtomicBoolean(false);
@@ -15,16 +15,16 @@ public class BlockCacheLocation {
         _block = block;
     }
 
-    public void setBankId(int bankId) {
-        _bankId = bankId;
+    public void setSlabId(int slabId) {
+        _slabId = slabId;
     }
 
     public int getBlock() {
         return _block;
     }
 
-    public int getBankId() {
-        return _bankId;
+    public int getSlabId() {
+        return _slabId;
     }
 
     public void touch() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
index 91c28f1..93eda49 100644
--- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
+++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
@@ -1,7 +1,5 @@
 package com.nearinfinity.blur.store;
 
-import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION;
-
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
@@ -35,6 +33,8 @@ import com.nearinfinity.blur.store.blockcache.BlockDirectoryCache;
 import com.nearinfinity.blur.store.hdfs.HdfsDirectory;
 import com.nearinfinity.blur.store.lock.BlurLockFactory;
 
+import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION;
+
 public class BenchmarkDirectory {
 
   public static void main(String[] args) throws IOException {
@@ -119,10 +119,10 @@ public class BenchmarkDirectory {
     return document;
   }
 
-  public static int getNumberOfBanks(float heapPercentage, int numberOfBlocksPerBank, int
blockSize) {
+  public static int getNumberOfSlabs(float heapPercentage, int numberOfBlocksPerSlab, int
blockSize) {
     long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
     long targetBytes = (long) (max * heapPercentage);
-    int slabSize = numberOfBlocksPerBank * blockSize;
+    int slabSize = numberOfBlocksPerSlab * blockSize;
     int slabs = (int) (targetBytes / slabSize);
     if (slabs == 0) {
       throw new RuntimeException("Minimum heap size is 512m!");

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/172a058a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
index 7573e8b..ad1923d 100644
--- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
+++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
@@ -1,7 +1,5 @@
 package com.nearinfinity.blur.store;
 
-import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION;
-
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.Arrays;
@@ -32,6 +30,8 @@ import com.nearinfinity.blur.store.blockcache.BlockDirectory;
 import com.nearinfinity.blur.store.blockcache.BlockDirectoryCache;
 import com.nearinfinity.blur.store.hdfs.HdfsDirectory;
 
+import static com.nearinfinity.blur.lucene.LuceneConstant.LUCENE_VERSION;
+
 public class BenchmarkDirectoryNrt {
 
   public static void main(String[] args) throws IOException, InterruptedException {
@@ -131,10 +131,10 @@ public class BenchmarkDirectoryNrt {
     return document;
   }
 
-  public static int getNumberOfBanks(float heapPercentage, int numberOfBlocksPerBank, int
blockSize) {
+  public static int getNumberOfSlabs(float heapPercentage, int numberOfBlocksPerSlab, int
blockSize) {
     long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
     long targetBytes = (long) (max * heapPercentage);
-    int slabSize = numberOfBlocksPerBank * blockSize;
+    int slabSize = numberOfBlocksPerSlab * blockSize;
     int slabs = (int) (targetBytes / slabSize);
     if (slabs == 0) {
       throw new RuntimeException("Minimum heap size is 512m!");


Mime
View raw message