incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: More updates, making the quiet option during merges not effect the cache.
Date Tue, 07 Jan 2014 15:42:33 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 01b4c7606 -> 7968b8f6b


More updates, making the quiet option during merges not effect the cache.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7968b8f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7968b8f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7968b8f6

Branch: refs/heads/apache-blur-0.2
Commit: 7968b8f6b1bed16a24717e4f0cada96fc47fc572
Parents: 01b4c76
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Jan 7 10:41:12 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Jan 7 10:41:12 2014 -0500

----------------------------------------------------------------------
 .../store/blockcache_v2/CacheIndexInput.java    | 46 +++++++++++++++++---
 .../store/blockcache_v2/CacheIndexOutput.java   | 22 ++++++----
 .../cachevalue/UnsafeCacheValue.java            |  5 ++-
 .../src/main/resources/blur-default.properties  |  2 +-
 4 files changed, 58 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
index df81832..35a03e6 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
@@ -38,6 +38,7 @@ public class CacheIndexInput extends IndexInput {
   private IndexInput _indexInput;
   private CacheKey _key = new CacheKey();
   private CacheValue _cacheValue;
+  private CacheValue _cacheValueQuietRef;
 
   private long _position;
   private int _blockPosition;
@@ -210,6 +211,11 @@ public class CacheIndexInput extends IndexInput {
       _indexInput.close();
       releaseCache();
     }
+    if (_cacheValueQuietRef != null) {
+      CacheValue ref = _cacheValueQuietRef;
+      _cacheValueQuietRef = null;
+      ref.release();
+    }
   }
 
   @Override
@@ -260,6 +266,7 @@ public class CacheIndexInput extends IndexInput {
     if (clone._cacheValue != null) {
       clone._cacheValue.incRef();
     }
+    clone._cacheValueQuietRef = null;
     return clone;
   }
 
@@ -295,9 +302,37 @@ public class CacheIndexInput extends IndexInput {
     }
   }
 
-  private void fill() throws IOException {
+  private void fillQuietly() throws IOException {
     _key.setBlockId(getBlockId());
-    _cacheValue = get(_key);
+    _cacheValue = _cache.getQuietly(_key);
+    if (_cacheValue == null) {
+      if (_cacheValueQuietRef == null) {
+        _cacheValueQuietRef = _cache.newInstance(_directory, _fileName);
+      }
+      _cacheValue = _cacheValueQuietRef;
+      _cacheValue.incRef();
+      long filePosition = getFilePosition();
+      _indexInput.seek(filePosition);
+      byte[] buffer = _store.takeBuffer(_bufferSize);
+      int len = (int) Math.min(_cacheBlockSize, _fileLength - filePosition);
+      int cachePosition = 0;
+      while (len > 0) {
+        int length = Math.min(_bufferSize, len);
+        _indexInput.readBytes(buffer, 0, length);
+        _cacheValue.write(cachePosition, buffer, 0, length);
+        len -= length;
+        cachePosition += length;
+      }
+      _store.putBuffer(buffer);
+    } else {
+      _cacheValue.incRef();
+    }
+    _blockPosition = getBlockPosition();
+  }
+
+  private void fillNormally() throws IOException {
+    _key.setBlockId(getBlockId());
+    _cacheValue = _cache.get(_key);
     if (_cacheValue == null) {
       _cacheValue = _cache.newInstance(_directory, _fileName);
       _cacheValue.incRef();
@@ -321,11 +356,12 @@ public class CacheIndexInput extends IndexInput {
     _blockPosition = getBlockPosition();
   }
 
-  private CacheValue get(CacheKey key) {
+  private void fill() throws IOException {
     if (_quiet) {
-      return _cache.getQuietly(key);
+      fillQuietly();
+    } else {
+      fillNormally();
     }
-    return _cache.get(key);
   }
 
   private int getBlockPosition() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
index 7c15cf2..dd0dc29 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
@@ -33,6 +33,7 @@ public class CacheIndexOutput extends IndexOutput {
   private final int _fileBufferSize;
   private final int _cacheBlockSize;
   private final Store _store;
+  private final boolean _shouldBeQuiet;
 
   private long _position;
   private byte[] _buffer;
@@ -49,6 +50,7 @@ public class CacheIndexOutput extends IndexOutput {
     _indexOutput = indexOutput;
     _store = BufferStore.instance(_cacheBlockSize);
     _buffer = _store.takeBuffer(_cacheBlockSize);
+    _shouldBeQuiet = _cache.shouldBeQuiet(directory, fileName);
   }
 
   @Override
@@ -89,19 +91,21 @@ public class CacheIndexOutput extends IndexOutput {
   }
 
   private void flushInternal() throws IOException {
-    int length = _cacheBlockSize - remaining();
+    final int length = _cacheBlockSize - remaining();
     if (length == 0) {
       return;
     }
-    CacheValue cacheValue = _cache.newInstance(_directory, _fileName);
-    cacheValue.incRef();
-    writeBufferToOutputStream(length);
-    cacheValue.write(0, _buffer, 0, length);
-    long blockId = (_position - length) / _cacheBlockSize;
-    cacheValue = cacheValue.trim(length);
-    _cache.put(new CacheKey(_fileId, blockId), cacheValue);
-    cacheValue.decRef();
+    if (!_shouldBeQuiet) {
+      CacheValue cacheValue = _cache.newInstance(_directory, _fileName);
+      cacheValue.incRef();
+      cacheValue.write(0, _buffer, 0, length);
+      long blockId = (_position - length) / _cacheBlockSize;
+      cacheValue = cacheValue.trim(length);
+      _cache.put(new CacheKey(_fileId, blockId), cacheValue);
+      cacheValue.decRef();
+    }
     _bufferPosition = 0;
+    writeBufferToOutputStream(length);
   }
 
   private void writeBufferToOutputStream(int len) throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
index 5aa4436..6f230fd 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
@@ -96,8 +96,9 @@ public class UnsafeCacheValue extends BaseCacheValue {
       _unsafe.freeMemory(_address);
       _released = true;
       _offHeapMemorySize.addAndGet(0 - _capacity);
-    } else {
-      new Throwable().printStackTrace();
+//    } else {
+      // @TODO this is here to debug against double releases.
+//      new Throwable().printStackTrace();
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7968b8f6/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index e0d6180..c0f71c3 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -146,7 +146,7 @@ blur.shard.time.between.commits=30000
 blur.shard.time.between.refreshs=3000
 
 # The max number of threads used during index merges
-blur.shard.merge.thread.count=3
+blur.shard.merge.thread.count=8
 
 # The maximum number of clauses in a BooleanQuery
 blur.max.clause.count=1024


Mime
View raw message