incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cr...@apache.org
Subject [19/30] git commit: Updating the block cache to be configurable
Date Thu, 07 Nov 2013 02:41:42 GMT
Updating the block cache to be configurable


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8424d1e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8424d1e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8424d1e8

Branch: refs/heads/blur-console-v2
Commit: 8424d1e8f778548afe74b1a362fb35eac98aa635
Parents: 3543160
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Oct 23 14:35:33 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Oct 23 14:35:33 2013 -0400

----------------------------------------------------------------------
 .../store/BlockCacheDirectoryFactoryV2.java     | 107 +++++++++++++++++--
 .../blur/store/blockcache_v2/BaseCache.java     |  26 +++--
 .../store/blockcache_v2/CacheDirectory.java     |  24 ++++-
 .../store/blockcache_v2/CacheIndexOutput.java   |   3 +-
 .../blur/store/blockcache_v2/CacheValue.java    |  35 ++++++
 .../cachevalue/BaseCacheValue.java              |  16 +++
 .../cachevalue/ByteArrayCacheValue.java         |  12 +++
 .../cachevalue/UnsafeCacheValue.java            |  18 +++-
 .../blur/store/CacheDirectoryTestSuite.java     |   2 +-
 .../store/blockcache_v2/CacheDirectoryTest.java |   2 +-
 .../apache/blur/metrics/MetricsConstants.java   |   1 +
 .../org/apache/blur/utils/BlurConstants.java    |  14 +++
 .../src/main/resources/blur-default.properties  |  31 ++++++
 13 files changed, 264 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
index 840d0cd..759b12c 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
@@ -16,12 +16,31 @@
  */
 package org.apache.blur.store;
 
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_STORE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_V2_WRITE_NOCACHE_EXT;
+import static org.apache.blur.utils.BlurConstants.DEFAULT_VALUE;
+import static org.apache.blur.utils.BlurConstants.OFF_HEAP;
 import static org.apache.blur.utils.BlurConstants.SHARED_MERGE_SCHEDULER;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
 import org.apache.blur.store.blockcache_v2.BaseCache;
 import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
 import org.apache.blur.store.blockcache_v2.Cache;
@@ -33,12 +52,56 @@ import org.apache.lucene.store.Directory;
 
 public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory {
 
-  private Cache _cache;
+  private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV2.class);
+
+  private final Cache _cache;
 
   public BlockCacheDirectoryFactoryV2(BlurConfiguration configuration, long totalNumberOfBytes)
{
-    final int fileBufferSizeInt = 8192;
-    final int cacheBlockSizeInt = 8192;
-    final STORE store = STORE.OFF_HEAP;
+
+    final int fileBufferSizeInt = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE,
8192);
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE, fileBufferSizeInt);
+    final int cacheBlockSizeInt = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE,
8192);
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE, cacheBlockSizeInt);
+
+    final Map<String,Integer> cacheBlockSizeMap = new HashMap<String, Integer>();
+    Map<String, String> properties = configuration.getProperties();
+    for (Entry<String, String> prop : properties.entrySet()) {
+      String key = prop.getKey();
+      if (key.startsWith(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX)) {
+        String value = prop.getValue();
+        int cacheBlockSizeForFile = Integer.parseInt(value);
+        String fieldType = key.substring(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX.length());
+        
+        cacheBlockSizeMap.put(fieldType, cacheBlockSizeForFile);
+        LOG.info("{0}={1} for file type [{2}]", key, cacheBlockSizeForFile, fieldType);
+      }
+    }
+
+    final STORE store = STORE.valueOf(configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_STORE,
OFF_HEAP));
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_STORE, store);
+
+    final Set<String> cachingFileExtensionsForRead = getSet(configuration.get(
+        BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT, DEFAULT_VALUE));
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT, cachingFileExtensionsForRead);
+    
+    final Set<String> nonCachingFileExtensionsForRead = getSet(configuration.get(
+        BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT, DEFAULT_VALUE));
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT, nonCachingFileExtensionsForRead);
+    
+    final boolean defaultReadCaching = configuration.getBoolean(BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT,
true);
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT, defaultReadCaching);
+
+    final Set<String> cachingFileExtensionsForWrite = getSet(configuration.get(
+        BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT, DEFAULT_VALUE));
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT, cachingFileExtensionsForWrite);
+    
+    final Set<String> nonCachingFileExtensionsForWrite = getSet(configuration.get(
+        BLUR_SHARD_BLOCK_CACHE_V2_WRITE_NOCACHE_EXT, DEFAULT_VALUE));
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_WRITE_NOCACHE_EXT, nonCachingFileExtensionsForWrite);
+    
+    final boolean defaultWriteCaching = configuration.getBoolean(BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT,
true);
+    LOG.info("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT, defaultWriteCaching);
+    
 
     Size fileBufferSize = new Size() {
       @Override
@@ -50,6 +113,11 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
     Size cacheBlockSize = new Size() {
       @Override
       public int getSize(CacheDirectory directory, String fileName) {
+        String ext = getExt(fileName);
+        Integer size = cacheBlockSizeMap.get(ext);
+        if (size != null) {
+          return size;
+        }
         return cacheBlockSizeInt;
       }
     };
@@ -57,20 +125,26 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
     FileNameFilter readFilter = new FileNameFilter() {
       @Override
       public boolean accept(CacheDirectory directory, String fileName) {
-        if (fileName.endsWith(".fdt")) {
+        String ext = getExt(fileName);
+        if (cachingFileExtensionsForRead.contains(ext)) {
+          return true;
+        } else if (nonCachingFileExtensionsForRead.contains(ext)) {
           return false;
         }
-        return true;
+        return defaultReadCaching;
       }
     };
 
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(CacheDirectory directory, String fileName) {
-        if (fileName.endsWith(".fdt")) {
+        String ext = getExt(fileName);
+        if (cachingFileExtensionsForWrite.contains(ext)) {
+          return true;
+        } else if (nonCachingFileExtensionsForWrite.contains(ext)) {
           return false;
         }
-        return true;
+        return defaultWriteCaching;
       }
     };
 
@@ -89,10 +163,23 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet, store);
   }
 
+  private Set<String> getSet(String value) {
+    String[] split = value.split(",");
+    return new HashSet<String>(Arrays.asList(split));
+  }
+
   @Override
-  public Directory newDirectory(String table, String shard, Directory directory, Set<String>
blockCacheFileTypes)
+  public Directory newDirectory(String table, String shard, Directory directory, Set<String>
tableBlockCacheFileTypes)
       throws IOException {
-    return new CacheDirectory(table, shard, directory, _cache);
+    return new CacheDirectory(table, shard, directory, _cache, tableBlockCacheFileTypes);
+  }
+
+  private static String getExt(String fileName) {
+    int indexOf = fileName.lastIndexOf('.');
+    if (indexOf < 0) {
+      return DEFAULT_VALUE;
+    }
+    return fileName.substring(indexOf + 1);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
index 4d7a2bb..6534e9a 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
@@ -28,6 +28,7 @@ import static org.apache.blur.metrics.MetricsConstants.SIZE;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
@@ -58,7 +59,7 @@ public class BaseCache extends Cache implements Closeable {
 
   private static final Log LOG = LogFactory.getLog(BaseCache.class);
   private static final long _1_MINUTE = TimeUnit.MINUTES.toMillis(1);
-  protected static final long _1_SECOND = TimeUnit.SECONDS.toMillis(1);
+  protected static final long _10_SECOND = TimeUnit.SECONDS.toMillis(10);
 
   public enum STORE {
     ON_HEAP, OFF_HEAP
@@ -86,7 +87,7 @@ public class BaseCache extends Cache implements Closeable {
 
     @Override
     public String toString() {
-      return "ReleaseEntry [_key=" + _key + ", _value=" + _value + "]";
+      return "ReleaseEntry [_key=" + _key + ", _value=" + _value + ", _createTime=" + _createTime
+ "]";
     }
 
     public boolean hasLivedToLong(long warningTimeForEntryCleanup) {
@@ -106,6 +107,7 @@ public class BaseCache extends Cache implements Closeable {
   private final Size _cacheBlockSize;
   private final Size _fileBufferSize;
   private final Map<FileIdKey, Long> _fileNameToId = new ConcurrentHashMap<FileIdKey,
Long>();
+  private final Map<Long, FileIdKey> _oldFileNameIdMap = new ConcurrentHashMap<Long,
FileIdKey>();
   private final AtomicLong _fileId = new AtomicLong();
   private final Quiet _quiet;
   private final Meter _hits;
@@ -116,7 +118,7 @@ public class BaseCache extends Cache implements Closeable {
   private final Thread _oldCacheValueDaemonThread;
   private final AtomicBoolean _running = new AtomicBoolean(true);
   private final BlockingQueue<ReleaseEntry> _releaseQueue;
-  private final long _warningTimeForEntryCleanup = TimeUnit.MINUTES.toMillis(1);
+  private final long _warningTimeForEntryCleanup = TimeUnit.MINUTES.toMillis(60);
 
   public BaseCache(long totalNumberOfBytes, Size fileBufferSize, Size cacheBlockSize, FileNameFilter
readFilter,
       FileNameFilter writeFilter, Quiet quiet, STORE store) {
@@ -169,7 +171,7 @@ public class BaseCache extends Cache implements Closeable {
         while (_running.get()) {
           cleanupOldCacheValues();
           try {
-            Thread.sleep(_1_SECOND);
+            Thread.sleep(_10_SECOND);
           } catch (InterruptedException e) {
             return;
           }
@@ -183,8 +185,14 @@ public class BaseCache extends Cache implements Closeable {
 
   protected void cleanupOldCacheValues() {
     Iterator<ReleaseEntry> iterator = _releaseQueue.iterator();
+    Map<Long, FileIdKey> entriesToCleanup = new HashMap<Long, FileIdKey>(_oldFileNameIdMap);
     while (iterator.hasNext()) {
       ReleaseEntry entry = iterator.next();
+      CacheKey key = entry._key;
+      long fileId = key.getFileId();
+      // Still referenced
+      entriesToCleanup.remove(fileId);
+
       CacheValue value = entry._value;
       if (value.refCount() == 0) {
         value.release();
@@ -192,9 +200,14 @@ public class BaseCache extends Cache implements Closeable {
         long capacity = _cacheMap.capacity();
         _cacheMap.setCapacity(capacity + value.size());
       } else if (entry.hasLivedToLong(_warningTimeForEntryCleanup)) {
-        LOG.warn("CacheValue has not been released [{0}] for over [{1} ms]", entry, _warningTimeForEntryCleanup);
+        FileIdKey fileIdKey = _oldFileNameIdMap.get(fileId);
+        LOG.warn("CacheValue has not been released [{0}] for [{1}] for over [{2} ms]", entry,
fileIdKey,
+            _warningTimeForEntryCleanup);
       }
     }
+    for (Long l : entriesToCleanup.keySet()) {
+      _oldFileNameIdMap.remove(l);
+    }
   }
 
   protected void cleanupOldFiles() {
@@ -236,7 +249,7 @@ public class BaseCache extends Cache implements Closeable {
       releaseEntry._key = key;
       releaseEntry._value = value;
 
-      LOG.info("CacheValue was not released [{0}]", releaseEntry);
+      LOG.debug("CacheValue was not released [{0}]", releaseEntry);
       _releaseQueue.add(releaseEntry);
     }
   }
@@ -267,6 +280,7 @@ public class BaseCache extends Cache implements Closeable {
     }
     long newId = _fileId.incrementAndGet();
     _fileNameToId.put(cachedFileName, newId);
+    _oldFileNameIdMap.put(newId, cachedFileName);
     return newId;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
index 8d71912..92af756 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
@@ -19,6 +19,7 @@ package org.apache.blur.store.blockcache_v2;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Set;
 
 import org.apache.blur.store.blockcache.LastModified;
 import org.apache.blur.store.hdfs.DirectoryDecorator;
@@ -36,8 +37,10 @@ public class CacheDirectory extends Directory implements DirectoryDecorator,
Las
   private final Cache _cache;
   private final String _shard;
   private final String _table;
+  private final Set<String> _tableBlockCacheFileTypes;
 
-  public CacheDirectory(String table, String shard, Directory directory, Cache cache) {
+  public CacheDirectory(String table, String shard, Directory directory, Cache cache,
+      Set<String> tableBlockCacheFileTypes) {
     if (!(directory instanceof LastModified)) {
       throw new RuntimeException("Directory [" + directory + "] does not implement '" + LastModified.class.toString()
           + "'");
@@ -47,6 +50,7 @@ public class CacheDirectory extends Directory implements DirectoryDecorator,
Las
     _directoryName = notNull(table + "_" + shard);
     _internal = notNull(directory);
     _cache = notNull(cache);
+    _tableBlockCacheFileTypes = tableBlockCacheFileTypes;
   }
 
   public String getShard() {
@@ -64,15 +68,29 @@ public class CacheDirectory extends Directory implements DirectoryDecorator,
Las
 
   public IndexInput openInput(String name, IOContext context) throws IOException {
     IndexInput indexInput = _internal.openInput(name, context);
-    if (_cache.cacheFileForReading(this, name, context)) {
+    if (_cache.cacheFileForReading(this, name, context) || isCachableFile(name)) {
       return new CacheIndexInput(this, name, indexInput, _cache);
     }
     return indexInput;
   }
 
+  private boolean isCachableFile(String name) {
+    if (_tableBlockCacheFileTypes == null) {
+      return true;
+    } else if (_tableBlockCacheFileTypes.isEmpty()) {
+      return false;
+    }
+    for (String ext : _tableBlockCacheFileTypes) {
+      if (name.endsWith(ext)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   public IndexOutput createOutput(String name, IOContext context) throws IOException {
     IndexOutput indexOutput = _internal.createOutput(name, context);
-    if (_cache.cacheFileForWriting(this, name, context)) {
+    if (_cache.cacheFileForWriting(this, name, context) || isCachableFile(name)) {
       return new CacheIndexOutput(this, name, indexOutput, _cache);
     }
     return indexOutput;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
index 9e33986..959e267 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
@@ -50,7 +50,7 @@ public class CacheIndexOutput extends IndexOutput {
 
   @Override
   public void setLength(long length) throws IOException {
-    
+
   }
 
   @Override
@@ -94,6 +94,7 @@ public class CacheIndexOutput extends IndexOutput {
     writeBufferToOutputStream(length);
     cacheValue.write(0, _buffer, 0, length);
     long blockId = (_position - length) / _cacheBlockSize;
+    cacheValue = cacheValue.trim(length);
     _cache.put(new CacheKey(_fileId, blockId), cacheValue);
     _bufferPosition = 0;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheValue.java
index c5c7759..68da92f 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheValue.java
@@ -17,6 +17,8 @@
  */
 package org.apache.blur.store.blockcache_v2;
 
+import javax.swing.text.Position;
+
 public interface CacheValue {
 
   /**
@@ -90,10 +92,43 @@ public interface CacheValue {
    */
   void release();
 
+  /**
+   * Reads a short from the given position.
+   * 
+   * @param position
+   *          the {@link Position} to read from.
+   * @return the short.
+   */
   short readShort(int position);
 
+  /**
+   * Reads a int from the given position.
+   * 
+   * @param position
+   *          the {@link Position} to read from.
+   * @return the int.
+   */
   int readInt(int position);
 
+  /**
+   * Reads a long from the given position.
+   * 
+   * @param position
+   *          the {@link Position} to read from.
+   * @return the long.
+   */
   long readLong(int position);
 
+  /**
+   * This method will trim the existing {@link CacheValue} and produce
+   * potentially a new {@link CacheValue} with the same data up to the length
+   * provided. Also if a new {@link CacheValue} is produced then this method is
+   * responsible to calling release on the old {@link CacheValue}.
+   * 
+   * @param length
+   *          the valid amount of data in the {@link CacheValue}.
+   * @return new trim {@link CacheValue} that has been trimmed if needed.
+   */
+  CacheValue trim(int length);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/BaseCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/BaseCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/BaseCacheValue.java
index 5c6136f..319e42d 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/BaseCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/BaseCacheValue.java
@@ -17,16 +17,31 @@
  */
 package org.apache.blur.store.blockcache_v2.cachevalue;
 
+import static org.apache.blur.metrics.MetricsConstants.CACHE_VALUE_FINALIZE;
+import static org.apache.blur.metrics.MetricsConstants.JVM;
+import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR;
+
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.blur.metrics.AtomicLongGauge;
 import org.apache.blur.store.blockcache_v2.CacheValue;
 
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.MetricName;
+
 @SuppressWarnings("serial")
 public abstract class BaseCacheValue extends AtomicLong implements CacheValue {
 
   private final int _length;
   protected volatile boolean _released = false;
 
+  private static final AtomicLong _neededFinalizedCall = new AtomicLong();
+
+  static {
+    Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, JVM, CACHE_VALUE_FINALIZE), new AtomicLongGauge(
+        _neededFinalizedCall));
+  }
+
   public BaseCacheValue(int length) {
     _length = length;
   }
@@ -123,6 +138,7 @@ public abstract class BaseCacheValue extends AtomicLong implements CacheValue
{
     // @TODO this may not be needed.
     if (!_released) {
       release();
+      _neededFinalizedCall.incrementAndGet();
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/ByteArrayCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/ByteArrayCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/ByteArrayCacheValue.java
index 5f4db4f..c1d53cc 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/ByteArrayCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/ByteArrayCacheValue.java
@@ -17,6 +17,8 @@
  */
 package org.apache.blur.store.blockcache_v2.cachevalue;
 
+import org.apache.blur.store.blockcache_v2.CacheValue;
+
 @SuppressWarnings("serial")
 public class ByteArrayCacheValue extends BaseCacheValue {
 
@@ -51,4 +53,14 @@ public class ByteArrayCacheValue extends BaseCacheValue {
   public int size() {
     return length();
   }
+
+  @Override
+  public CacheValue trim(int length) {
+    if (_buffer.length == length) {
+      return this;
+    }
+    ByteArrayCacheValue cacheValue = new ByteArrayCacheValue(length);
+    System.arraycopy(_buffer, 0, cacheValue._buffer, 0, length);
+    return cacheValue;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
index c954316..603fb11 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
@@ -25,6 +25,7 @@ import java.lang.reflect.Field;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.blur.metrics.AtomicLongGauge;
+import org.apache.blur.store.blockcache_v2.CacheValue;
 
 import sun.misc.Unsafe;
 
@@ -67,15 +68,11 @@ public class UnsafeCacheValue extends BaseCacheValue {
 
   public UnsafeCacheValue(int length) {
     super(length);
-    _capacity = getCapacity(length);
+    _capacity = length;
     _address = _unsafe.allocateMemory(_capacity);
     _offHeapMemorySize.addAndGet(_capacity);
   }
 
-  private int getCapacity(int length) {
-    return length;
-  }
-
   @Override
   protected void writeInternal(int position, byte[] buf, int offset, int length) {
     copyFromArray(buf, offset, length, resolveAddress(position));
@@ -110,4 +107,15 @@ public class UnsafeCacheValue extends BaseCacheValue {
   public int size() {
     return _capacity;
   }
+
+  @Override
+  public CacheValue trim(int length) {
+    if (length == _capacity) {
+      return this;
+    }
+    UnsafeCacheValue unsafeCacheValue = new UnsafeCacheValue(length);
+    _unsafe.copyMemory(_address, unsafeCacheValue._address, length);
+    release();
+    return unsafeCacheValue;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
index 87fcaad..9960d96 100644
--- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
@@ -76,7 +76,7 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
     Directory dir = FSDirectory.open(new File(file, "cache"));
 
     BufferStore.init(128, 128);
-    return new CacheDirectory("test", "test", wrapLastModified(dir), _cache);
+    return new CacheDirectory("test", "test", wrapLastModified(dir), _cache, null);
   }
 
   protected abstract STORE getStore();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
index b6b615a..800191f 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
@@ -91,7 +91,7 @@ public class CacheDirectoryTest {
         STORE.ON_HEAP);
     Directory directory = newDirectory();
     BufferStore.init(128, 128);
-    _cacheDirectory = new CacheDirectory("test", "test", directory, _cache);
+    _cacheDirectory = new CacheDirectory("test", "test", directory, _cache, null);
   }
 
   @After

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java b/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java
index 71024be..48eca37 100644
--- a/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/metrics/MetricsConstants.java
@@ -60,4 +60,5 @@ public class MetricsConstants {
   public static final String MERGE_THROUGHPUT_BYTES = "Merge Throughput Bytes";
   public static final String GC_TIMES = "GC Times (ms)";
   public static final String OFF_HEAP_MEMORY = "Off Heap Memory";
+  public static final String CACHE_VALUE_FINALIZE = "CacheValue Finalize";
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index d58d304..07b5d7a 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -78,6 +78,20 @@ public class BlurConstants {
   public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread";
   public static final String BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS = "blur.shard.distributed.layout.factory.class";
   
+
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT = "blur.shard.block.cache.v2.read.cache.ext";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT = "blur.shard.block.cache.v2.read.nocache.ext";
+  public static final String DEFAULT_VALUE = "";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT = "blur.shard.block.cache.v2.write.cache.ext";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_WRITE_NOCACHE_EXT = "blur.shard.block.cache.v2.write.nocache.ext";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT = "blur.shard.block.cache.v2.write.default";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT = "blur.shard.block.cache.v2.read.default";
+  public static final String OFF_HEAP = "OFF_HEAP";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_STORE = "blur.shard.block.cache.v2.store";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX = "blur.shard.block.cache.v2.cacheBlockSize.";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE = "blur.shard.block.cache.v2.fileBufferSize";
+  public static final String BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE = "blur.shard.block.cache.v2.cacheBlockSize";
+  
   public static final String BLUR_FIELDTYPE = "blur.fieldtype.";
 
   public static final String BLUR_SHARD_TIME_BETWEEN_COMMITS = "blur.shard.time.between.commits";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8424d1e8/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index fdae2e7..ec58c78 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -81,6 +81,37 @@ blur.shard.blockcache.direct.memory.allocation=true
 # v1 version of block cache only. The slabs in the blockcache are automatically configured
by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through the MaxDirectoryMemorySize
provided to the JVM
 blur.shard.blockcache.slab.count=-1
 
+# v2 version of block cache only. File buffer size, this is the buffer size used to read
and write to data to HDFS.  For production this will likely be increased.
+blur.shard.block.cache.v2.fileBufferSize=8192
+
+# v2 version of block cache only. The is the size of the blocks in the off heap cache, it
is good practice to have this match 'blur.shard.block.cache.v2.fileBufferSize'.  For production
this will likely be increased.
+blur.shard.block.cache.v2.cacheBlockSize=8192
+
+# v2 version of block cache only. Specific files type can have their own block cache sizes,
such as filter files that should have a very large mapping. 
+#blur.shard.block.cache.v2.cacheBlockSize.<ext>=
+blur.shard.block.cache.v2.cacheBlockSize.filter=33554432
+
+# v2 version of block cache only. This is used to control if the block are created on or
off heap.  Values are OFF_HEAP | ON_HEAP
+blur.shard.block.cache.v2.store=OFF_HEAP
+
+# v2 version of block cache only. This specifies what file types should be cached during
reads.  Comma delimited list.
+blur.shard.block.cache.v2.read.cache.ext=
+
+# v2 version of block cache only. This specifies what file types should NOT be cached during
reads.  Comma delimited list.
+blur.shard.block.cache.v2.read.nocache.ext=fdt
+
+# v2 version of block cache only. This specifies the default behavior if a file type is not
specified in the cache or nocache lists during reads.  Values true | false
+blur.shard.block.cache.v2.read.default=true
+
+# v2 version of block cache only. This specifies what file types should be cached during
writes.  Comma delimited list.
+blur.shard.block.cache.v2.write.cache.ext=
+
+# v2 version of block cache only. This specifies what file types should NOT be cached during
writes.  Comma delimited list.
+blur.shard.block.cache.v2.write.nocache.ext=fdt
+
+# v2 version of block cache only. This specifies the default behavior if a file type is not
specified in the cache or nocache lists during writes.  Values true | false
+blur.shard.block.cache.v2.write.default=true
+
 # The number of 1K byte buffers
 blur.shard.buffercache.1024=8192
 


Mime
View raw message