incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Adding local file handle cache lookup system. This will reduce load on the the main block cache map and improve performance.
Date Mon, 29 Jun 2015 12:06:24 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 7f837cb84 -> a71e2fe84


Adding local file handle cache lookup system.  This will reduce load on the the main block
cache map and improve performance.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a6eecd40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a6eecd40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a6eecd40

Branch: refs/heads/master
Commit: a6eecd4035cb2dff80e964e79afac488a0622e3d
Parents: 1c7aba2
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Jun 29 08:06:09 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Jun 29 08:06:09 2015 -0400

----------------------------------------------------------------------
 .../store/BlockCacheDirectoryFactoryV2.java     |  62 +++++---
 .../blur/store/blockcache_v2/BaseCache.java     |  46 ++++--
 .../apache/blur/store/blockcache_v2/Cache.java  |  16 ++
 .../store/blockcache_v2/CacheIndexInput.java    |  22 ++-
 .../blockcache_v2/DirectIndexInputCache.java    |  51 +++++++
 .../store/blockcache_v2/IndexInputCache.java    |  49 +++++++
 .../store/blockcache_v2/LRUIndexInputCache.java |  48 ++++++
 .../blur/store/blockcache_v2/MeterWrapper.java  | 145 +++++++++++++++++++
 .../blur/store/blockcache_v2/PooledCache.java   |   6 +
 .../blur/store/CacheDirectoryTestSuite.java     |  11 +-
 .../blur/store/blockcache_v2/BaseCacheTest.java |  10 +-
 .../CacheDirectoryLoadUnloadTest.java           |   9 +-
 .../store/blockcache_v2/CacheDirectoryTest.java |  10 +-
 .../blockcache_v2/CacheIndexInputTest.java      |   5 +
 .../store/blockcache_v2/MeterWrapperTest.java   |  66 +++++++++
 .../store/blockcache_v2/NoIndexInputCache.java  |  38 +++++
 .../org/apache/blur/utils/BlurConstants.java    |   6 +-
 .../src/main/resources/blur-default.properties  |   6 +
 .../apache/blur/utils/BlurConstantsTest.java    |  11 +-
 19 files changed, 570 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
index 627c73e..a700b37 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
@@ -18,6 +18,8 @@ package org.apache.blur.store;
 
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT_PREFIX;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_POOL_CACHE_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT;
@@ -62,8 +64,6 @@ import org.apache.lucene.store.Directory;
 
 public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory {
 
-
-
   private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV2.class);
 
   private final Cache _cache;
@@ -74,23 +74,14 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
     LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE, fileBufferSizeInt);
     final int cacheBlockSizeInt = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE,
8192);
     LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE, cacheBlockSizeInt);
+    final int directLocalCacheRefLimitInt = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT,
+        64 * 1024 * 1024);
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT, directLocalCacheRefLimitInt);
 
-    final Map<String, Integer> cacheBlockSizeMap = new HashMap<String, Integer>();
-    Map<String, String> properties = configuration.getProperties();
-    for (Entry<String, String> prop : properties.entrySet()) {
-      String key = prop.getKey();
-      String value = prop.getValue();
-      if (value == null || value.isEmpty()) {
-        continue;
-      }
-      if (key.startsWith(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX)) {
-        int cacheBlockSizeForFile = Integer.parseInt(value);
-        String fieldType = key.substring(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX.length());
-
-        cacheBlockSizeMap.put(fieldType, cacheBlockSizeForFile);
-        LOG.info("{0}={1} for file type [{2}]", key, cacheBlockSizeForFile, fieldType);
-      }
-    }
+    final Map<String, Integer> cacheBlockSizeMap = getIntMap(configuration,
+        BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX);
+    final Map<String, Integer> directLocalCacheRefLimitMap = getIntMap(configuration,
+        BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT_PREFIX);
 
     final STORE store = STORE.valueOf(configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_STORE,
OFF_HEAP));
     LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_STORE, store);
@@ -136,6 +127,18 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
       }
     };
 
+    Size directLocalCacheRefLimit = new Size() {
+      @Override
+      public int getSize(CacheDirectory directory, String fileName) {
+        String ext = getExt(fileName);
+        Integer size = directLocalCacheRefLimitMap.get(ext);
+        if (size != null) {
+          return size;
+        }
+        return directLocalCacheRefLimitInt;
+      }
+    };
+
     FileNameFilter readFilter = new FileNameFilter() {
       @Override
       public boolean accept(CacheDirectory directory, String fileName) {
@@ -184,12 +187,31 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
       pool = new SimpleCacheValueBufferPool(store, queueDepth);
     }
 
-    BaseCache baseCache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize,
readFilter, writeFilter,
-        quiet, pool);
+    BaseCache baseCache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize,
directLocalCacheRefLimit,
+        readFilter, writeFilter, quiet, pool);
     CachePoolStrategy cachePoolStrategy = new SingleCachePoolStrategy(baseCache);
     _cache = new PooledCache(cachePoolStrategy);
   }
 
+  private Map<String, Integer> getIntMap(BlurConfiguration configuration, String prefix)
{
+    final Map<String, Integer> map = new HashMap<String, Integer>();
+    Map<String, String> properties = configuration.getProperties();
+    for (Entry<String, String> prop : properties.entrySet()) {
+      String key = prop.getKey();
+      String value = prop.getValue();
+      if (value == null || value.isEmpty()) {
+        continue;
+      }
+      if (key.startsWith(prefix)) {
+        int i = Integer.parseInt(value);
+        String fieldType = key.substring(prefix.length());
+        map.put(fieldType, i);
+        LOG.info("{0}={1} for file type [{2}]", key, i, fieldType);
+      }
+    }
+    return map;
+  }
+
   private Set<String> getSet(String value) {
     String[] split = value.split(",");
     return new HashSet<String>(Arrays.asList(split));

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
index 3bb7904..9cd0e98 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
@@ -43,6 +43,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.store.blockcache_v2.cachevalue.DetachableCacheValue;
+import org.apache.commons.io.IOUtils;
 import org.apache.lucene.store.IOContext;
 
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
@@ -50,7 +51,6 @@ import com.googlecode.concurrentlinkedhashmap.EvictionListener;
 import com.googlecode.concurrentlinkedhashmap.Weigher;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Gauge;
-import com.yammer.metrics.core.Meter;
 import com.yammer.metrics.core.MetricName;
 
 public class BaseCache extends Cache implements Closeable {
@@ -87,30 +87,34 @@ public class BaseCache extends Cache implements Closeable {
   private final FileNameFilter _writeFilter;
   private final Size _cacheBlockSize;
   private final Size _fileBufferSize;
+  private final Size _directLocalCacheRefLimit;
   private final Map<FileIdKey, Long> _fileNameToId = new ConcurrentHashMap<FileIdKey,
Long>();
   private final AtomicLong _fileId = new AtomicLong();
   private final Quiet _quiet;
-  private final Meter _hits;
-  private final Meter _misses;
-  private final Meter _evictions;
-  private final Meter _removals;
+  private final MeterWrapper _hits;
+  private final MeterWrapper _misses;
+  private final MeterWrapper _evictions;
+  private final MeterWrapper _removals;
   private final Thread _oldFileDaemonThread;
   private final AtomicBoolean _running = new AtomicBoolean(true);
   private final BaseCacheValueBufferPool _cacheValueBufferPool;
 
-  public BaseCache(long totalNumberOfBytes, Size fileBufferSize, Size cacheBlockSize, FileNameFilter
readFilter,
-      FileNameFilter writeFilter, Quiet quiet, BaseCacheValueBufferPool cacheValueBufferPool)
{
+  public BaseCache(long totalNumberOfBytes, Size fileBufferSize, Size cacheBlockSize, Size
directLocalCacheRefLimit,
+      FileNameFilter readFilter, FileNameFilter writeFilter, Quiet quiet, BaseCacheValueBufferPool
cacheValueBufferPool) {
     _cacheMap = new ConcurrentLinkedHashMap.Builder<CacheKey, CacheValue>().weigher(new
BaseCacheWeigher())
         .maximumWeightedCapacity(totalNumberOfBytes).listener(new BaseCacheEvictionListener()).build();
     _fileBufferSize = fileBufferSize;
     _readFilter = readFilter;
     _writeFilter = writeFilter;
     _cacheBlockSize = cacheBlockSize;
+    _directLocalCacheRefLimit = directLocalCacheRefLimit;
     _quiet = quiet;
-    _hits = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, HIT), HIT, TimeUnit.SECONDS);
-    _misses = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, MISS), MISS, TimeUnit.SECONDS);
-    _evictions = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, EVICTION), EVICTION,
TimeUnit.SECONDS);
-    _removals = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, REMOVAL), REMOVAL,
TimeUnit.SECONDS);
+    _hits = MeterWrapper.wrap(Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, HIT),
HIT, TimeUnit.SECONDS));
+    _misses = MeterWrapper.wrap(Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE, MISS),
MISS, TimeUnit.SECONDS));
+    _evictions = MeterWrapper.wrap(Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE,
EVICTION), EVICTION,
+        TimeUnit.SECONDS));
+    _removals = MeterWrapper.wrap(Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, CACHE,
REMOVAL), REMOVAL,
+        TimeUnit.SECONDS));
     _cacheValueBufferPool = cacheValueBufferPool;
     Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, CACHE, ENTRIES), new Gauge<Long>()
{
       @Override
@@ -143,6 +147,10 @@ public class BaseCache extends Cache implements Closeable {
     _oldFileDaemonThread.start();
   }
 
+  public MeterWrapper getHitsMeter() {
+    return _hits;
+  }
+
   public int getEntryCount() {
     return _cacheMap.size();
   }
@@ -172,6 +180,10 @@ public class BaseCache extends Cache implements Closeable {
     closeCachMap();
     _oldFileDaemonThread.interrupt();
     _cacheValueBufferPool.close();
+    IOUtils.closeQuietly(_evictions);
+    IOUtils.closeQuietly(_hits);
+    IOUtils.closeQuietly(_misses);
+    IOUtils.closeQuietly(_removals);
   }
 
   private void closeCachMap() {
@@ -350,4 +362,16 @@ public class BaseCache extends Cache implements Closeable {
     }
   }
 
+  @Override
+  public IndexInputCache createIndexInputCache(CacheDirectory directory, String fileName,
long fileLength) {
+    int cacheBlockSize = getCacheBlockSize(directory, fileName);
+    int limit = _directLocalCacheRefLimit.getSize(directory, fileName);
+    if (fileLength > limit) {
+      int entries = limit / cacheBlockSize;
+      return new LRUIndexInputCache(fileLength, cacheBlockSize, entries, _hits);
+    } else {
+      return new DirectIndexInputCache(fileLength, cacheBlockSize, _hits);
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
index 8a59141..d58ff92 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
@@ -186,4 +186,20 @@ public abstract class Cache implements Closeable {
    */
   public abstract void fileClosedForWriting(CacheDirectory directory, String fileName, long
fileId) throws IOException;
 
+  /**
+   * This method creates a local index input cache per file handle to try and
+   * reduce load on the main block cache lookup system.
+   * 
+   * 
+   * @param directory
+   *          the directory.
+   * @param fileName
+   *          the file name.
+   * @param fileLength
+   *          the file length.
+   * 
+   * @return the IndexInputCache instance.
+   */
+  public abstract IndexInputCache createIndexInputCache(CacheDirectory directory, String
fileName, long fileLength);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
index 4fe1a40..18b9eda 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
@@ -36,6 +36,7 @@ public class CacheIndexInput extends IndexInput {
   private final String _fileName;
   private final Cache _cache;
   private final Store _store;
+  private final IndexInputCache _indexInputCache;
 
   private IndexInput _indexInput;
   private CacheKey _key = new CacheKey();
@@ -58,6 +59,7 @@ public class CacheIndexInput extends IndexInput {
 
     _fileId = _cache.getFileId(_directory, _fileName);
     _cacheBlockSize = _cache.getCacheBlockSize(_directory, _fileName);
+    _indexInputCache = _cache.createIndexInputCache(_directory, _fileName, _fileLength);
     _bufferSize = _cache.getFileBufferSize(_directory, _fileName);
     _quiet = _cache.shouldBeQuiet(_directory, _fileName);
     _key.setFileId(_fileId);
@@ -297,6 +299,7 @@ public class CacheIndexInput extends IndexInput {
       _isClosed = true;
       _indexInput.close();
       releaseCache();
+      _indexInputCache.close();
     }
   }
 
@@ -390,7 +393,7 @@ public class CacheIndexInput extends IndexInput {
 
   private void fillQuietly() throws IOException {
     _key.setBlockId(getBlockId());
-    _cacheValue = _cache.getQuietly(_directory, _fileName, _key);
+    _cacheValue = lookup(true);
     if (_cacheValue == null) {
       if (_cacheValueQuietRefCannotBeReleased == null) {
         // @TODO this could be improved.
@@ -417,7 +420,7 @@ public class CacheIndexInput extends IndexInput {
 
   private void fillNormally() throws IOException {
     _key.setBlockId(getBlockId());
-    _cacheValue = _cache.get(_directory, _fileName, _key);
+    _cacheValue = lookup(false);
     if (_cacheValue == null) {
       _cacheValue = _cache.newInstance(_directory, _fileName);
       long filePosition = getFilePosition();
@@ -438,6 +441,21 @@ public class CacheIndexInput extends IndexInput {
     _blockPosition = getBlockPosition();
   }
 
+  private CacheValue lookup(boolean quietly) {
+    CacheValue cacheValue = _indexInputCache.get(_key.getBlockId());
+    if (cacheValue == null) {
+      if (quietly) {
+        cacheValue = _cache.getQuietly(_directory, _fileName, _key);
+      } else {
+        cacheValue = _cache.get(_directory, _fileName, _key);
+      }
+    }
+    if (cacheValue != null) {
+      _indexInputCache.put(_key.getBlockId(), cacheValue);
+    }
+    return cacheValue;
+  }
+
   private void fill() throws IOException {
     if (_quiet) {
       fillQuietly();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/DirectIndexInputCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/DirectIndexInputCache.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/DirectIndexInputCache.java
new file mode 100644
index 0000000..f08bbcf
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/DirectIndexInputCache.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2;
+
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+public class DirectIndexInputCache extends IndexInputCache {
+
+  private final AtomicReferenceArray<CacheValue> _cacheValues;
+
+  public DirectIndexInputCache(long fileLength, int cacheBlockSize, MeterWrapper hits) {
+    super(fileLength, cacheBlockSize, hits);
+    int indexLength = (int) (fileLength / cacheBlockSize);
+    if (fileLength % cacheBlockSize != 0) {
+      indexLength++;
+    }
+    _cacheValues = new AtomicReferenceArray<CacheValue>(indexLength);
+  }
+
+  @Override
+  public void put(long blockId, CacheValue cacheValue) {
+    _cacheValues.set((int) blockId, cacheValue);
+  }
+
+  @Override
+  public CacheValue get(long blockId) {
+    int id = (int) blockId;
+    CacheValue cacheValue = _cacheValues.get(id);
+    if (cacheValue == null) {
+      return null;
+    } else if (cacheValue.isEvicted()) {
+      _cacheValues.compareAndSet(id, cacheValue, null);
+      return null;
+    }
+    return cacheValue;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/IndexInputCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/IndexInputCache.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/IndexInputCache.java
new file mode 100644
index 0000000..e29806b
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/IndexInputCache.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+
+public abstract class IndexInputCache implements Closeable {
+
+  protected final long _fileLength;
+  protected final int _cacheBlockSize;
+  protected final MeterWrapper _hits;
+
+  protected IndexInputCache(long fileLength, int cacheBlockSize, MeterWrapper hits) {
+    _fileLength = fileLength;
+    _cacheBlockSize = cacheBlockSize;
+    _hits = hits;
+  }
+
+  public abstract void put(long blockId, CacheValue cacheValue);
+
+  public abstract CacheValue get(long blockId);
+
+  protected void hit() {
+    _hits.mark();
+  }
+
+  @Override
+  public void close() throws IOException {
+    IOUtils.closeQuietly(_hits);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/LRUIndexInputCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/LRUIndexInputCache.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/LRUIndexInputCache.java
new file mode 100644
index 0000000..e4a5497
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/LRUIndexInputCache.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+
+public class LRUIndexInputCache extends IndexInputCache {
+
+  private final ConcurrentLinkedHashMap<Long, CacheValue> _cacheMap;
+
+  public LRUIndexInputCache(long fileLength, int cacheBlockSize, int entries, MeterWrapper
hits) {
+    super(fileLength, cacheBlockSize, hits);
+    _cacheMap = new ConcurrentLinkedHashMap.Builder<Long, CacheValue>().maximumWeightedCapacity(entries).build();
+  }
+
+  @Override
+  public void put(long blockId, CacheValue cacheValue) {
+    _cacheMap.put(blockId, cacheValue);
+  }
+
+  @Override
+  public CacheValue get(long blockId) {
+    CacheValue cacheValue = _cacheMap.get(blockId);
+    if (cacheValue == null) {
+      return null;
+    } else if (cacheValue.isEvicted()) {
+      _cacheMap.remove(blockId, cacheValue);
+      return null;
+    }
+    hit();
+    return cacheValue;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/MeterWrapper.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/MeterWrapper.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/MeterWrapper.java
new file mode 100644
index 0000000..d608e14
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/MeterWrapper.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.yammer.metrics.core.Meter;
+
+public abstract class MeterWrapper implements Closeable {
+
+  public static final MeterWrapper NOTHING = new MeterWrapper() {
+    @Override
+    public void mark() {
+
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+  };
+
+  private final static Timer _timer;
+  private final static ConcurrentMap<String, MeterWrapperCounter> _counterMap = new
ConcurrentHashMap<String, MeterWrapperCounter>();
+
+  private static final long DELAY = TimeUnit.SECONDS.toMillis(1);
+
+  static {
+    _timer = new Timer("MeterWrapper", true);
+    _timer.schedule(getCounterUnLoadTimerTask(), DELAY, DELAY);
+  }
+
+  public static interface SimpleMeter {
+    void mark(long l);
+  }
+
+  public static MeterWrapper wrap(final Meter meter) {
+    return wrap(new SimpleMeter() {
+      @Override
+      public void mark(long l) {
+        meter.mark(l);
+      }
+    });
+  }
+
+  public static MeterWrapper wrap(final SimpleMeter meter) {
+    final String id = UUID.randomUUID().toString();
+    final ThreadLocal<AtomicLong> countThreadLocal = new ThreadLocal<AtomicLong>()
{
+      @Override
+      protected AtomicLong initialValue() {
+        AtomicLong counter = new AtomicLong();
+        register(id, meter, counter);
+        return counter;
+      }
+    };
+    return new MeterWrapper() {
+      @Override
+      public void mark() {
+        countThreadLocal.get().incrementAndGet();
+      }
+
+      @Override
+      public void close() throws IOException {
+        unregister(id);
+      }
+    };
+  }
+
+  private static TimerTask getCounterUnLoadTimerTask() {
+    return new TimerTask() {
+      @Override
+      public void run() {
+        updateMetrics();
+      }
+    };
+  }
+
+  public static void updateMetrics() {
+    Collection<MeterWrapperCounter> values = _counterMap.values();
+    for (MeterWrapperCounter meterWrapperCounter : values) {
+      meterWrapperCounter.markMeter();
+    }
+  }
+
+  private static void unregister(String id) {
+    _counterMap.remove(id);
+  }
+
+  private static void register(String id, SimpleMeter meter, AtomicLong counter) {
+    {
+      _counterMap.putIfAbsent(id, new MeterWrapperCounter(meter));
+    }
+    {
+      _counterMap.get(id).add(counter);
+    }
+  }
+
+  static class MeterWrapperCounter {
+    final SimpleMeter _meter;
+    final Set<AtomicLong> _counterSet = Collections.newSetFromMap(new ConcurrentHashMap<AtomicLong,
Boolean>());
+
+    MeterWrapperCounter(SimpleMeter meter) {
+      _meter = meter;
+    }
+
+    void add(AtomicLong counter) {
+      _counterSet.add(counter);
+    }
+
+    void markMeter() {
+      for (AtomicLong count : _counterSet) {
+        _meter.mark(count.getAndSet(0));
+      }
+    }
+
+  }
+
+  public abstract void mark();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java
index ef554f3..aac7150 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/PooledCache.java
@@ -117,4 +117,10 @@ public class PooledCache extends Cache {
     return _cachePoolStrategy;
   }
 
+  @Override
+  public IndexInputCache createIndexInputCache(CacheDirectory directory, String fileName,
long fileLength) {
+    Cache cache = _cachePoolStrategy.getCache(directory, fileName);
+    return cache.createIndexInputCache(directory, fileName, fileLength);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
index 664ab06..855b860 100644
--- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
@@ -35,7 +35,7 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
 
   @Override
   protected Directory setupDirectory() throws IOException {
-    int totalNumberOfBytes = 1000000;
+    final int totalNumberOfBytes = 1000000;
     final int fileBufferSizeInt = numberBetween(113, 215);
     final int cacheBlockSizeInt = numberBetween(111, 251);
 
@@ -52,6 +52,13 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
         return cacheBlockSizeInt;
       }
     };
+    
+    Size directLocalCacheLimit = new Size() {
+      @Override
+      public int getSize(CacheDirectory directory, String fileName) {
+        return totalNumberOfBytes;
+      }
+    };
 
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
@@ -71,7 +78,7 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
         return false;
       }
     };
-    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
+    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, directLocalCacheLimit,
readFilter, writeFilter, quiet,
         getPool());
     Directory dir = FSDirectory.open(new File(file, "cache"));
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
index dca0e1b..fa70e1f 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/BaseCacheTest.java
@@ -32,7 +32,7 @@ public class BaseCacheTest {
 
   @Test
   public void test1() {
-    int totalNumberOfBytes = 1000000;
+    final int totalNumberOfBytes = 1000000;
     final int fileBufferSizeInt = 127;
     final int cacheBlockSizeInt = 131;
     Size fileBufferSize = new Size() {
@@ -47,6 +47,12 @@ public class BaseCacheTest {
         return cacheBlockSizeInt;
       }
     };
+    Size directLocalCacheLimit = new Size() {
+      @Override
+      public int getSize(CacheDirectory directory, String fileName) {
+        return totalNumberOfBytes;
+      }
+    };
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(CacheDirectory directory, String fileName) {
@@ -66,7 +72,7 @@ public class BaseCacheTest {
       }
     };
     SimpleCacheValueBufferPool simpleCacheValueBufferPool = new SimpleCacheValueBufferPool(STORE.ON_HEAP,
1000);
-    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
+    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, directLocalCacheLimit,
readFilter, writeFilter, quiet,
         simpleCacheValueBufferPool);
 
     Directory directory = newDirectory();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
index f9da8f4..cbef07a 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryLoadUnloadTest.java
@@ -38,6 +38,7 @@ public class CacheDirectoryLoadUnloadTest {
   // @Before
   public void setup() throws IOException {
     long totalNumberOfBytes = 2000000000L;
+    final int directLocalCacheLimitBytes = 64 * 1024 * 1024;
     final int fileBufferSizeInt = 8192;
     final int cacheBlockSizeInt = 8192;
     Size fileBufferSize = new Size() {
@@ -52,6 +53,12 @@ public class CacheDirectoryLoadUnloadTest {
         return cacheBlockSizeInt;
       }
     };
+    Size directLocalCacheLimit = new Size() {
+      @Override
+      public int getSize(CacheDirectory directory, String fileName) {
+        return directLocalCacheLimitBytes;
+      }
+    };
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(CacheDirectory directory, String fileName) {
@@ -71,7 +78,7 @@ public class CacheDirectoryLoadUnloadTest {
       }
     };
     SimpleCacheValueBufferPool simpleCacheValueBufferPool = new SimpleCacheValueBufferPool(STORE.OFF_HEAP,
1000);
-    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
+    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, directLocalCacheLimit,
readFilter, writeFilter, quiet,
         simpleCacheValueBufferPool);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
index 6f0ff40..434f8a1 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
@@ -54,7 +54,7 @@ public class CacheDirectoryTest {
 
   @Before
   public void setup() {
-    int totalNumberOfBytes = 1000000;
+    final int totalNumberOfBytes = 1000000;
     final int fileBufferSizeInt = 127;
     final int cacheBlockSizeInt = 131;
     Size fileBufferSize = new Size() {
@@ -69,6 +69,12 @@ public class CacheDirectoryTest {
         return cacheBlockSizeInt;
       }
     };
+    Size directLocalCacheLimit = new Size() {
+      @Override
+      public int getSize(CacheDirectory directory, String fileName) {
+        return totalNumberOfBytes;
+      }
+    };
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(CacheDirectory directory, String fileName) {
@@ -88,7 +94,7 @@ public class CacheDirectoryTest {
       }
     };
     SimpleCacheValueBufferPool simpleCacheValueBufferPool = new SimpleCacheValueBufferPool(STORE.ON_HEAP,
1000);
-    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
+    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, directLocalCacheLimit,
readFilter, writeFilter, quiet,
         simpleCacheValueBufferPool);
     Directory directory = newDirectory();
     BufferStore.initNewBuffer(1024, 1024 * 128);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
index ed50696..ef5e470 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
@@ -265,6 +265,11 @@ public class CacheIndexInputTest {
 
       }
 
+      @Override
+      public IndexInputCache createIndexInputCache(CacheDirectory directory, String fileName,
long fileLength) {
+        return NoIndexInputCache.instance();
+      }
+
     };
     return cacheFactory;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/MeterWrapperTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/MeterWrapperTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/MeterWrapperTest.java
new file mode 100644
index 0000000..5398745
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/MeterWrapperTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.blur.store.blockcache_v2.MeterWrapper.SimpleMeter;
+import org.junit.Test;
+
+public class MeterWrapperTest {
+
+  @Test
+  public void test() throws InterruptedException {
+    final AtomicLong totalCount = new AtomicLong();
+    SimpleMeter simpleMeter = new MeterWrapper.SimpleMeter() {
+      @Override
+      public void mark(long l) {
+        totalCount.addAndGet(l);
+      }
+    };
+    final MeterWrapper wrap = MeterWrapper.wrap(simpleMeter);
+    int threadCount = 4;
+    final int max = Integer.MAX_VALUE / 10;
+    List<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < threadCount; i++) {
+      Thread thread = new Thread(new Runnable() {
+        @Override
+        public void run() {
+          for (int l = 0; l < max; l++) {
+            wrap.mark();
+          }
+        }
+      });
+      threads.add(thread);
+    }
+    long s = System.nanoTime();
+    for (Thread t : threads) {
+      t.start();
+    }
+    for (Thread t : threads) {
+      t.join();
+    }
+    long e = System.nanoTime();
+    System.out.println("Time [" + (e - s) / 1000000.0 + " ms]");
+    MeterWrapper.updateMetrics();
+    assertEquals(max * 4, totalCount.get());
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/NoIndexInputCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/NoIndexInputCache.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/NoIndexInputCache.java
new file mode 100644
index 0000000..68d5ed6
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/NoIndexInputCache.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.store.blockcache_v2;
+
+public class NoIndexInputCache {
+
+  private static final IndexInputCache NONE = new IndexInputCache(0, 0, null) {
+
+    @Override
+    public void put(long blockId, CacheValue cacheValue) {
+
+    }
+
+    @Override
+    public CacheValue get(long blockId) {
+      return null;
+    }
+  };
+
+  public static IndexInputCache instance() {
+    return NONE;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 24488e1..955d6cf 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -99,7 +99,7 @@ public class BlurConstants {
   public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes";
   public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread";
   public static final String BLUR_SHARD_DEEP_PAGING_CACHE_SIZE = "blur.shard.deep.paging.cache.size";
-  
+
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_POOL_CACHE_SIZE = "blur.shard.block.cache.v2.pool.cache.size";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_SLAB_CHUNK_SIZE = "blur.shard.block.cache.v2.slab.chunk.size";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_SLAB_SIZE = "blur.shard.block.cache.v2.slab.size";
@@ -114,14 +114,16 @@ public class BlurConstants {
   public static final String OFF_HEAP = "OFF_HEAP";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_STORE = "blur.shard.block.cache.v2.store";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX = "blur.shard.block.cache.v2.cacheBlockSize.";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT_PREFIX = "blur.shard.block.cache.v2.directRefLimit.";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE = "blur.shard.block.cache.v2.fileBufferSize";
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE = "blur.shard.block.cache.v2.cacheBlockSize";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT = "blur.shard.block.cache.v2.directRefLimit";
   public static final String BLUR_SHARD_BLURINDEX_CLASS = "blur.shard.blurindex.class";
   public static final String BLUR_SHARD_SERVER_MINIMUM_BEFORE_SAFEMODE_EXIT = "blur.shard.server.minimum.before.safemode.exit";
   public static final String BLUR_SHARD_SMALL_MERGE_THRESHOLD = "blur.shard.small.merge.threshold";
   public static final String BLUR_SHARD_REQUEST_CACHE_SIZE = "blur.shard.request.cache.size";
   public static final String BLUR_GC_BACK_PRESSURE_HEAP_RATIO = "blur.gc.back.pressure.heap.ratio";
-  
+
   public static final String BLUR_SHARD_DEFAULT_READ_SEQUENTIAL_SKIP_THRESHOLD = "blur.shard.default.read.sequential.skip.threshold";
   public static final String BLUR_SHARD_DEFAULT_READ_SEQUENTIAL_THRESHOLD = "blur.shard.default.read.sequential.threshold";
   public static final String BLUR_SHARD_MERGE_READ_SEQUENTIAL_SKIP_THRESHOLD = "blur.shard.merge.read.sequential.skip.threshold";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index e2988c4..999389b 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -152,6 +152,12 @@ blur.shard.block.cache.v2.cacheBlockSize=8192
 # v2 version of block cache only. Specific files type can have their own block cache sizes,
such as filter files that should have a very large mapping. 
 blur.shard.block.cache.v2.cacheBlockSize.<ext>=
 
+# v2 version of block cache only. The is the max size of file that can have a direct reference
cache in the file handle.
+blur.shard.block.cache.v2.directRefLimit=67108864
+
+# v2 version of block cache only. The is the max size of file that can have a direct reference
cache in the file handle per file extension. 
+blur.shard.block.cache.v2.directRefLimit.<ext>=
+
 # v2 version of block cache only. This is used to control if the block are created on or
off heap.  Values are OFF_HEAP | ON_HEAP
 blur.shard.block.cache.v2.store=OFF_HEAP
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a6eecd40/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java
----------------------------------------------------------------------
diff --git a/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java b/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java
index 96eab53..1cad148 100644
--- a/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java
+++ b/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java
@@ -48,11 +48,12 @@ public class BlurConstantsTest {
     _emptyDefaultProperties.addAll(Arrays.asList("BLUR_HDFS_TRACE_PATH", "BLUR_SHARD_HOSTNAME",
         "BLUR_SHARD_BLOCK_CACHE_TOTAL_SIZE", "BLUR_CONTROLLER_HOSTNAME", "BLUR_SHARD_READ_INTERCEPTOR",
         "BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT", "BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT",
-        "BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX", "BLUR_FIELDTYPE", "BLUR_SHARD_FILTERED_SERVER_CLASS",
-        "BLUR_CONTROLLER_FILTERED_SERVER_CLASS", "BLUR_COMMAND_LIB_PATH", "BLUR_TMP_PATH",
"BLUR_SECURITY_SASL_TYPE",
-        "BLUR_SECUTIRY_SASL_CUSTOM_CLASS", "BLUR_SECURITY_SASL_LDAP_DOMAIN", "BLUR_SECURITY_SASL_LDAP_BASEDN",
-        "BLUR_SECURITY_SASL_LDAP_URL", "BLUR_SERVER_SECURITY_FILTER_CLASS", "BLUR_FILTER_ALIAS",
-        "BLUR_BULK_UPDATE_WORKING_PATH", "BLUR_BULK_UPDATE_WORKING_PATH_PERMISSION"));
+        "BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX", "BLUR_SHARD_BLOCK_CACHE_V2_DIRECT_REF_LIMIT_PREFIX",
+        "BLUR_FIELDTYPE", "BLUR_SHARD_FILTERED_SERVER_CLASS", "BLUR_CONTROLLER_FILTERED_SERVER_CLASS",
+        "BLUR_COMMAND_LIB_PATH", "BLUR_TMP_PATH", "BLUR_SECURITY_SASL_TYPE", "BLUR_SECUTIRY_SASL_CUSTOM_CLASS",
+        "BLUR_SECURITY_SASL_LDAP_DOMAIN", "BLUR_SECURITY_SASL_LDAP_BASEDN", "BLUR_SECURITY_SASL_LDAP_URL",
+        "BLUR_SERVER_SECURITY_FILTER_CLASS", "BLUR_FILTER_ALIAS", "BLUR_BULK_UPDATE_WORKING_PATH",
+        "BLUR_BULK_UPDATE_WORKING_PATH_PERMISSION"));
   }
 
   @Test


Mime
View raw message