incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Fixed BLUR-433
Date Mon, 15 Jun 2015 13:18:11 GMT
Fixed BLUR-433


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/90625d02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/90625d02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/90625d02

Branch: refs/heads/master
Commit: 90625d0235dd4d60a812be9fedfc12e0be1d817b
Parents: 6107046
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Jun 15 09:17:53 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Jun 15 09:17:53 2015 -0400

----------------------------------------------------------------------
 .../store/BlockCacheDirectoryFactoryV2.java     | 21 ++++-
 .../blur/store/blockcache_v2/BaseCache.java     |  9 +-
 .../SlabAllocationCacheValueBufferPool.java     | 34 +++++++-
 .../UnsafeWrapperMultiCacheValue.java           |  2 +-
 .../blur/store/BaseDirectoryTestSuite.java      |  2 +
 .../blur/store/CacheDirectoryTestSuite.java     |  6 +-
 .../store/CacheDirectoryTestSuiteOffHeap.java   | 33 -------
 .../store/CacheDirectoryTestSuiteOnHeap.java    | 33 -------
 .../CacheDirectoryTestSuiteSimpleOffHeap.java   | 35 ++++++++
 .../CacheDirectoryTestSuiteSimpleOnHeap.java    | 35 ++++++++
 .../store/CacheDirectoryTestSuiteSlabPool.java  | 38 +++++++++
 .../blur/store/blockcache_v2/BaseCacheTest.java |  4 +-
 .../CacheDirectoryLoadUnloadTest.java           |  9 +-
 .../store/blockcache_v2/CacheDirectoryTest.java |  3 +-
 .../SlabAllocationCacheValueBufferPoolTest.java | 68 ++++++++++++++-
 .../UnsafeWrapperMultiCacheValueTest.java       | 90 ++++++++++++++++++++
 .../org/apache/blur/utils/BlurConstants.java    |  6 +-
 .../src/main/resources/blur-default.properties  | 12 +++
 18 files changed, 346 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
index 7e1b7b6..627c73e 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
@@ -19,9 +19,13 @@ package org.apache.blur.store;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_POOL_CACHE_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_SLAB_CHUNK_SIZE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_SLAB_ENABLED;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_SLAB_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_STORE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT;
@@ -43,18 +47,23 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.store.blockcache_v2.BaseCache;
 import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
+import org.apache.blur.store.blockcache_v2.BaseCacheValueBufferPool;
 import org.apache.blur.store.blockcache_v2.Cache;
 import org.apache.blur.store.blockcache_v2.CacheDirectory;
 import org.apache.blur.store.blockcache_v2.CachePoolStrategy;
 import org.apache.blur.store.blockcache_v2.FileNameFilter;
 import org.apache.blur.store.blockcache_v2.PooledCache;
 import org.apache.blur.store.blockcache_v2.Quiet;
+import org.apache.blur.store.blockcache_v2.SimpleCacheValueBufferPool;
 import org.apache.blur.store.blockcache_v2.SingleCachePoolStrategy;
 import org.apache.blur.store.blockcache_v2.Size;
+import org.apache.blur.store.blockcache_v2.SlabAllocationCacheValueBufferPool;
 import org.apache.lucene.store.Directory;
 
 public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory {
 
+
+
   private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV2.class);
 
   private final Cache _cache;
@@ -165,8 +174,18 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
       }
     };
 
+    BaseCacheValueBufferPool pool;
+    if (configuration.getBoolean(BLUR_SHARD_BLOCK_CACHE_V2_SLAB_ENABLED, true)) {
+      int slabSize = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_SLAB_SIZE, 134217728);
+      int chunkSize = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_SLAB_CHUNK_SIZE, 8192);
+      pool = new SlabAllocationCacheValueBufferPool(chunkSize, slabSize);
+    } else {
+      int queueDepth = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_POOL_CACHE_SIZE, 10000);
+      pool = new SimpleCacheValueBufferPool(store, queueDepth);
+    }
+
     BaseCache baseCache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize,
readFilter, writeFilter,
-        quiet, store);
+        quiet, pool);
     CachePoolStrategy cachePoolStrategy = new SingleCachePoolStrategy(baseCache);
     _cache = new PooledCache(cachePoolStrategy);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
index 5dc473c..3bb7904 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
@@ -85,7 +85,6 @@ public class BaseCache extends Cache implements Closeable {
   private final ConcurrentLinkedHashMap<CacheKey, CacheValue> _cacheMap;
   private final FileNameFilter _readFilter;
   private final FileNameFilter _writeFilter;
-  private final STORE _store;
   private final Size _cacheBlockSize;
   private final Size _fileBufferSize;
   private final Map<FileIdKey, Long> _fileNameToId = new ConcurrentHashMap<FileIdKey,
Long>();
@@ -97,24 +96,22 @@ public class BaseCache extends Cache implements Closeable {
   private final Meter _removals;
   private final Thread _oldFileDaemonThread;
   private final AtomicBoolean _running = new AtomicBoolean(true);
-  private final SimpleCacheValueBufferPool _cacheValueBufferPool;
+  private final BaseCacheValueBufferPool _cacheValueBufferPool;
 
   public BaseCache(long totalNumberOfBytes, Size fileBufferSize, Size cacheBlockSize, FileNameFilter
readFilter,
-      FileNameFilter writeFilter, Quiet quiet, STORE store) {
+      FileNameFilter writeFilter, Quiet quiet, BaseCacheValueBufferPool cacheValueBufferPool)
{
     _cacheMap = new ConcurrentLinkedHashMap.Builder<CacheKey, CacheValue>().weigher(new
BaseCacheWeigher())
         .maximumWeightedCapacity(totalNumberOfBytes).listener(new BaseCacheEvictionListener()).build();
     _fileBufferSize = fileBufferSize;
     _readFilter = readFilter;
     _writeFilter = writeFilter;
-    _store = store;
     _cacheBlockSize = cacheBlockSize;
     _quiet = quiet;
     _hits = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, HIT), HIT, TimeUnit.SECONDS);
     _misses = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, MISS), MISS, TimeUnit.SECONDS);
     _evictions = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, EVICTION), EVICTION,
TimeUnit.SECONDS);
     _removals = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, REMOVAL), REMOVAL,
TimeUnit.SECONDS);
-    // @TODO make configurable
-    _cacheValueBufferPool = new SimpleCacheValueBufferPool(_store, 1000);
+    _cacheValueBufferPool = cacheValueBufferPool;
     Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, CACHE, ENTRIES), new Gauge<Long>()
{
       @Override
       public Long value() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPool.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPool.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPool.java
index 3b9a5c5..b59609b 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPool.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPool.java
@@ -55,18 +55,30 @@ public class SlabAllocationCacheValueBufferPool extends BaseCacheValueBufferPool
     final BlockLocks _locks;
     final int _chunkSize;
     final long _maxAddress;
+    final int _maxChunks;
 
-    Slab(long address, int numberOfChunks, int chunkSize) {
+    Slab(long address, int maxChunks, int chunkSize) {
+      _maxChunks = maxChunks;
       _address = address;
-      _maxAddress = _address + ((long) numberOfChunks * (long) chunkSize);
-      _locks = new BlockLocks(numberOfChunks);
+      _maxAddress = _address + ((long) maxChunks * (long) chunkSize);
+      _locks = new BlockLocks(maxChunks);
       _chunkSize = chunkSize;
     }
 
+    void dumpSlabPopulation() {
+      System.out.println("Address [" + _address + "] MaxAddress [" + _maxAddress + "] Clear:[");
+      int index = 0;
+      while ((index = _locks.nextClearBit(index)) != -1) {
+        System.out.println(index);
+        index++;
+      }
+      System.out.println("]");
+    }
+
     long findChunk() {
       while (true) {
         int chunkId = _locks.nextClearBit(0);
-        if (chunkId < 0) {
+        if (chunkId < 0 || chunkId>=_maxChunks) {
           return -1L;
         }
         if (_locks.set(chunkId)) {
@@ -87,6 +99,7 @@ public class SlabAllocationCacheValueBufferPool extends BaseCacheValueBufferPool
     void release() {
       _unsafe.freeMemory(_address);
     }
+
   }
 
   @Override
@@ -203,4 +216,17 @@ public class SlabAllocationCacheValueBufferPool extends BaseCacheValueBufferPool
     }
     return chunks + 1;
   }
+
+  public long getCurrentSize() {
+    Collection<Slab> slabs = getSlabs();
+    return (long) slabs.size() * (long) _slabSize;
+  }
+
+  public void dumpSlabPopulation() {
+    Collection<Slab> slabs = getSlabs();
+    for (Slab slab : slabs) {
+      slab.dumpSlabPopulation();
+    }
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValue.java
index ac4dfb6..bb7bb8a 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValue.java
@@ -56,7 +56,7 @@ public abstract class UnsafeWrapperMultiCacheValue extends UnsafeCacheValue
{
       int chunkPos = getChunkPosition(position);
       int len = Math.min(length, _chunkSize - chunkPos);
       long address = _chunkAddresses[chunkIndex];
-      copyToArray(resolveAddress(address, position), buf, offset, len);
+      copyToArray(resolveAddress(address, chunkPos), buf, offset, len);
       position += len;
       offset += len;
       length -= len;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
index dda70a0..fde60b0 100644
--- a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
+++ b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
@@ -81,6 +81,8 @@ public abstract class BaseDirectoryTestSuite {
     rm(file);
     rm(fileControl);
     seed = new Random().nextLong();
+    System.out.println("Seed [" + seed + "]");
+    seed = 4975561783806198199L;
     random = new Random(seed);
     directory = setupDirectory();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
index c342874..664ab06 100644
--- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
@@ -20,7 +20,7 @@ import java.io.File;
 import java.io.IOException;
 
 import org.apache.blur.store.blockcache_v2.BaseCache;
-import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
+import org.apache.blur.store.blockcache_v2.BaseCacheValueBufferPool;
 import org.apache.blur.store.blockcache_v2.CacheDirectory;
 import org.apache.blur.store.blockcache_v2.FileNameFilter;
 import org.apache.blur.store.blockcache_v2.Quiet;
@@ -72,7 +72,7 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
       }
     };
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
-        getStore());
+        getPool());
     Directory dir = FSDirectory.open(new File(file, "cache"));
 
     BufferStore.initNewBuffer(1024, 1024 * 128);
@@ -80,7 +80,7 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
     return new CacheDirectory("test", "test", wrapLastModified(dir), _cache, null);
   }
 
-  protected abstract STORE getStore();
+  protected abstract BaseCacheValueBufferPool getPool();
 
   public void close() throws IOException {
     _cache.close();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOffHeap.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOffHeap.java
b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOffHeap.java
deleted file mode 100644
index 6456765..0000000
--- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOffHeap.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
-import org.junit.Test;
-
-public class CacheDirectoryTestSuiteOffHeap extends CacheDirectoryTestSuite {
-
-  @Test
-  public void runsTheTests() {
-  }
-
-  @Override
-  protected STORE getStore() {
-    return STORE.OFF_HEAP;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOnHeap.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOnHeap.java
b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOnHeap.java
deleted file mode 100644
index 7130fa8..0000000
--- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteOnHeap.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
-import org.junit.Test;
-
-public class CacheDirectoryTestSuiteOnHeap extends CacheDirectoryTestSuite {
-
-  @Override
-  protected STORE getStore() {
-    return STORE.ON_HEAP;
-  }
-
-  @Test
-  public void runsTheTests() {
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOffHeap.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOffHeap.java
b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOffHeap.java
new file mode 100644
index 0000000..c0e9fed
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOffHeap.java
@@ -0,0 +1,35 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
+import org.apache.blur.store.blockcache_v2.BaseCacheValueBufferPool;
+import org.apache.blur.store.blockcache_v2.SimpleCacheValueBufferPool;
+import org.junit.Test;
+
+public class CacheDirectoryTestSuiteSimpleOffHeap extends CacheDirectoryTestSuite {
+
+  @Test
+  public void runsTheTests() {
+  }
+
+  @Override
+  protected BaseCacheValueBufferPool getPool() {
+    return new SimpleCacheValueBufferPool(STORE.OFF_HEAP, 1000);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOnHeap.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOnHeap.java
b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOnHeap.java
new file mode 100644
index 0000000..ec82820
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSimpleOnHeap.java
@@ -0,0 +1,35 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.blur.store.blockcache_v2.BaseCacheValueBufferPool;
+import org.apache.blur.store.blockcache_v2.SimpleCacheValueBufferPool;
+import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
+import org.junit.Test;
+
+public class CacheDirectoryTestSuiteSimpleOnHeap extends CacheDirectoryTestSuite {
+
+  @Test
+  public void runsTheTests() {
+  }
+
+  @Override
+  protected BaseCacheValueBufferPool getPool() {
+    return new SimpleCacheValueBufferPool(STORE.ON_HEAP, 1000);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSlabPool.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSlabPool.java
b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSlabPool.java
new file mode 100644
index 0000000..277dfc0
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuiteSlabPool.java
@@ -0,0 +1,38 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.blur.store.blockcache_v2.BaseCacheValueBufferPool;
+import org.apache.blur.store.blockcache_v2.SlabAllocationCacheValueBufferPool;
+import org.junit.Test;
+
+public class CacheDirectoryTestSuiteSlabPool extends CacheDirectoryTestSuite {
+
+  private int _slabSize;
+  private int _chunkSize = 1000;
+
+  @Test
+  public void runsTheTests() {
+  }
+
+  @Override
+  protected BaseCacheValueBufferPool getPool() {
+    _chunkSize = random.nextInt(50000) + 1000;
+    _slabSize = (random.nextInt(100) + 1) * _chunkSize;
+    return new SlabAllocationCacheValueBufferPool(_chunkSize, _slabSize);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
index 6457f87..dca0e1b 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
@@ -20,7 +20,6 @@ import java.io.IOException;
 
 import org.apache.blur.store.blockcache.LastModified;
 import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
-import org.apache.blur.store.blockcache_v2.CacheDirectoryTest.RDir;
 import org.apache.blur.store.buffer.BufferStore;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
@@ -66,8 +65,9 @@ public class BaseCacheTest {
         return false;
       }
     };
+    SimpleCacheValueBufferPool simpleCacheValueBufferPool = new SimpleCacheValueBufferPool(STORE.ON_HEAP,
1000);
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
-        STORE.ON_HEAP);
+        simpleCacheValueBufferPool);
 
     Directory directory = newDirectory();
     BufferStore.initNewBuffer(1024, 1024 * 128);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
index 751c2b9..f9da8f4 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
@@ -30,14 +30,12 @@ import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
 
 public class CacheDirectoryLoadUnloadTest {
 
   private BaseCache _cache;
 
-//  @Before
+  // @Before
   public void setup() throws IOException {
     long totalNumberOfBytes = 2000000000L;
     final int fileBufferSizeInt = 8192;
@@ -72,8 +70,9 @@ public class CacheDirectoryLoadUnloadTest {
         return false;
       }
     };
+    SimpleCacheValueBufferPool simpleCacheValueBufferPool = new SimpleCacheValueBufferPool(STORE.OFF_HEAP,
1000);
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
-        STORE.OFF_HEAP);
+        simpleCacheValueBufferPool);
   }
 
   @After
@@ -98,7 +97,7 @@ public class CacheDirectoryLoadUnloadTest {
 
   }
 
-//  @Test
+  // @Test
   public void test1() throws IOException {
     int i = 0;
     long length = 100000000;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
index c173aae..6f0ff40 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
@@ -87,8 +87,9 @@ public class CacheDirectoryTest {
         return false;
       }
     };
+    SimpleCacheValueBufferPool simpleCacheValueBufferPool = new SimpleCacheValueBufferPool(STORE.ON_HEAP,
1000);
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
-        STORE.ON_HEAP);
+        simpleCacheValueBufferPool);
     Directory directory = newDirectory();
     BufferStore.initNewBuffer(1024, 1024 * 128);
     BufferStore.initNewBuffer(8192, 8192 * 128);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPoolTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPoolTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPoolTest.java
index b3a77a9..d385c13 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPoolTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/SlabAllocationCacheValueBufferPoolTest.java
@@ -16,10 +16,15 @@
  */
 package org.apache.blur.store.blockcache_v2;
 
+import static org.junit.Assert.*;
+
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 
+import org.apache.blur.store.blockcache_v2.cachevalue.ByteArrayCacheValue;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -27,12 +32,14 @@ import org.junit.Test;
 public class SlabAllocationCacheValueBufferPoolTest {
 
   private SlabAllocationCacheValueBufferPool _pool;
+  private int _chunkSize;
+  private int _slabSize;
 
   @Before
   public void setup() {
-    int chunkSize = 1000;
-    int slabSize = chunkSize * 10000;
-    _pool = new SlabAllocationCacheValueBufferPool(chunkSize, slabSize);
+    _chunkSize = 1000;
+    _slabSize = _chunkSize * 10000;
+    _pool = new SlabAllocationCacheValueBufferPool(_chunkSize, _slabSize);
   }
 
   @After
@@ -46,17 +53,70 @@ public class SlabAllocationCacheValueBufferPoolTest {
     _pool.returnToPool(cacheValue1);
     CacheValue cacheValue2 = _pool.getCacheValue(1);
     _pool.returnToPool(cacheValue2);
+    assertEquals(_slabSize, _pool.getCurrentSize());
   }
 
   @Test
   public void test2() {
     List<CacheValue> list = new ArrayList<CacheValue>();
     int total = 10000;
+    int cacheBlockSize = 10000;
     for (int i = 0; i < total; i++) {
-      list.add(_pool.getCacheValue(10000));
+      list.add(_pool.getCacheValue(cacheBlockSize));
     }
+    assertEquals((long) total * (long) cacheBlockSize, _pool.getCurrentSize());
+    // _pool.dumpSlabPopulation();
+    list.add(_pool.getCacheValue(cacheBlockSize));
+    assertEquals(((long) total * (long) cacheBlockSize) + _slabSize, _pool.getCurrentSize());
     for (CacheValue cacheValue : list) {
       _pool.returnToPool(cacheValue);
     }
   }
+
+  @Test
+  public void test3() throws EvictionException {
+    Random random = new Random(1);
+    List<CacheValue> list = new ArrayList<CacheValue>();
+    List<CacheValue> copy = new ArrayList<CacheValue>();
+    int total = 10000;
+    int cacheBlockSize = 10000;
+    for (int i = 0; i < total; i++) {
+      CacheValue cacheValue = _pool.getCacheValue(cacheBlockSize);
+      ByteArrayCacheValue copyCv = new ByteArrayCacheValue(cacheBlockSize);
+      populate(random, cacheValue, copyCv);
+      list.add(cacheValue);
+      copy.add(copyCv);
+    }
+    assertEquals((long) total * (long) cacheBlockSize, _pool.getCurrentSize());
+    for (int i = 0; i < list.size(); i++) {
+      CacheValue cv1 = list.get(i);
+      CacheValue cv2 = copy.get(i);
+      equals(cv1, cv2);
+    }
+  }
+
+  private void equals(CacheValue cv1, CacheValue cv2) throws EvictionException {
+    int l1 = cv1.length();
+    int l2 = cv2.length();
+    assertEquals(l1, l2);
+
+    byte[] buf1 = new byte[l1];
+    byte[] buf2 = new byte[l2];
+    cv1.read(0, buf1, 0, l1);
+    cv2.read(0, buf2, 0, l2);
+
+    assertTrue(Arrays.equals(buf1, buf2));
+  }
+
+  private void populate(Random random, CacheValue cv1, CacheValue cv2) throws EvictionException
{
+    int l1 = cv1.length();
+    int l2 = cv2.length();
+    assertEquals(l1, l2);
+
+    byte[] buf1 = new byte[l1];
+    random.nextBytes(buf1);
+    cv1.write(0, buf1, 0, l1);
+    cv2.write(0, buf1, 0, l1);
+    equals(cv1, cv2);
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValueTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValueTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValueTest.java
new file mode 100644
index 0000000..fc2f255
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeWrapperMultiCacheValueTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2.cachevalue;
+
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.blur.store.blockcache_v2.EvictionException;
+import org.junit.Test;
+
+import sun.misc.Unsafe;
+
+public class UnsafeWrapperMultiCacheValueTest {
+
+  @Test
+  public void testUnsafeWrapperMultiCacheValue() throws EvictionException {
+
+    final Unsafe unsafe = UnsafeCacheValue._unsafe;
+    final long[] addresses = new long[10];
+    int chunkSize = 1000;
+    int chunks = 10;
+    for (int i = 0; i < chunks; i++) {
+      addresses[i] = unsafe.allocateMemory(chunkSize);
+    }
+
+    int length = addresses.length * chunkSize;
+
+    UnsafeWrapperMultiCacheValue unsafeWrapperMultiCacheValue = new UnsafeWrapperMultiCacheValue(length,
addresses,
+        chunkSize) {
+      @Override
+      protected void releaseInternal() {
+        for (int i = 0; i < 10; i++) {
+          unsafe.freeMemory(addresses[i]);
+        }
+      }
+    };
+
+    Random random = new Random();
+    byte[] buf1 = new byte[length];
+    random.nextBytes(buf1);
+
+    unsafeWrapperMultiCacheValue.write(0, buf1, 0, length);
+
+    byte[] buf2 = new byte[length];
+
+    unsafeWrapperMultiCacheValue.read(0, buf2, 0, length);
+
+    assertTrue(equals(buf1, buf2));
+    assertTrue(Arrays.equals(buf1, buf2));
+
+    unsafeWrapperMultiCacheValue.release();
+
+  }
+
+  public static boolean equals(byte[] a, byte[] a2) {
+    if (a == a2)
+      return true;
+    if (a == null || a2 == null)
+      return false;
+
+    int length = a.length;
+    if (a2.length != length)
+      return false;
+
+    for (int i = 0; i < length; i++)
+      if (a[i] != a2[i]) {
+        System.out.println("Pos [" + i + "]");
+        return false;
+      }
+
+    return true;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index fdf6cad..6d36711 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -98,7 +98,11 @@ public class BlurConstants {
   public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes";
   public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread";
   public static final String BLUR_SHARD_DEEP_PAGING_CACHE_SIZE = "blur.shard.deep.paging.cache.size";
-
+  
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_POOL_CACHE_SIZE = "blur.shard.block.cache.v2.pool.cache.size";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_SLAB_CHUNK_SIZE = "blur.shard.block.cache.v2.slab.chunk.size";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_SLAB_SIZE = "blur.shard.block.cache.v2.slab.size";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_SLAB_ENABLED = "blur.shard.block.cache.v2.slab.enabled";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT = "blur.shard.block.cache.v2.read.cache.ext";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT = "blur.shard.block.cache.v2.read.nocache.ext";
   public static final String DEFAULT_VALUE = "";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/90625d02/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 1682688..3a372fe 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -170,6 +170,18 @@ blur.shard.block.cache.v2.write.nocache.ext=fdt
 # v2 version of block cache only. This specifies the default behavior if a file type is not
specified in the cache or nocache lists during writes.  Values true | false
 blur.shard.block.cache.v2.write.default=true
 
+# v2 version of block cache only. This specifies size of the cache pool for the old non-slab
block cache pool.  If blur.shard.block.cache.v2.slab.enabled is true this value is ignored.
+blur.shard.block.cache.v2.pool.cache.size=10000
+
+# v2 version of block cache only. This specifies if slab allocation is enabled for v2 block
cache.  Values true | false
+blur.shard.block.cache.v2.slab.enabled=true
+
+# v2 version of block cache only. This specifies the chunk size for the cache, this is the
smallest allocation unit.
+blur.shard.block.cache.v2.slab.chunk.size=8192
+
+# v2 version of block cache only. This specifies the slab size for the cache is slab are
enabled.
+blur.shard.block.cache.v2.slab.size=134217728
+
 # Sets the compression of used in the storing of the fields. Valid entries are FAST FAST_DECOMPRESSION
HIGH_COMPRESSION
 blur.shard.index.compressionmode=FAST
 


Mime
View raw message