incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/6] git commit: More fixes to the v2 block cache.
Date Thu, 19 Sep 2013 13:07:46 GMT
More fixes to the v2 block cache.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/562537d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/562537d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/562537d1

Branch: refs/heads/master
Commit: 562537d140c7c6616e84ad85ea9f13b5ba4302c2
Parents: 239a56d
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Sep 4 19:28:47 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Sep 4 19:28:47 2013 -0400

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurShardServer.java      |  3 +-
 .../mapreduce/lib/BlurOutputFormatTest.java     |  3 +-
 .../store/BlockCacheDirectoryFactoryV2.java     |  7 ++--
 .../blur/store/blockcache_v2/BaseCache.java     |  2 +-
 .../store/blockcache_v2/CacheIndexInput.java    |  5 ++-
 .../store/blockcache_v2/CacheIndexOutput.java   | 36 ++++++++++++--------
 .../cachevalue/UnsafeCacheValue.java            |  7 ++--
 .../store/blockcache_v2/CacheDirectoryTest.java | 21 +++++++++---
 8 files changed, 56 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index c3f1af1..e1804d5 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -69,7 +69,7 @@ public class ThriftBlurShardServer extends ThriftServer {
 
   private static final Log LOG = LogFactory.getLog(ThriftBlurShardServer.class);
   private static final boolean enableJsonReporter = false;
-  private static final long _64MB = 0;
+  private static final long _64MB = 64 * 1024 * 1024;
 
   public static void main(String[] args) throws Exception {
     int serverIndex = getServerIndex(args);
@@ -115,6 +115,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     // Alternate BlockCacheDirectoryFactory support currently disabled in 0.2.0,
     // look for it in 0.2.1
     boolean experimentalBlockCache = configuration.getBoolean(BLUR_SHARD_EXPERIMENTAL_BLOCK_CACHE,
false);
+    experimentalBlockCache = false;
     if (!experimentalBlockCache) {
       // setup block cache
       // 134,217,728 is the slab size, therefore there are 16,384 blocks

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
index 6f936a7..faa3266 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
@@ -317,7 +317,8 @@ public class BlurOutputFormatTest {
 
   }
 
-  @Test
+  // @TODO this test to fail sometimes due to issues in the MR MiniCluster
+//  @Test
   public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException,
       ClassNotFoundException {
     localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
index c08c42a..e3a7007 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
@@ -46,13 +46,16 @@ public class BlockCacheDirectoryFactoryV2 implements BlockCacheDirectoryFactory
         if (fileName.endsWith(".fdt") || fileName.endsWith(".fdx")) {
           return false;
         }
-        return false;
+        return true;
       }
     };
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(String directoryName, String fileName) {
-        return false;
+        if (fileName.endsWith(".fdt") || fileName.endsWith(".fdx")) {
+          return false;
+        }
+        return true;
       }
     };
     _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, fileNameBlockSize, readFilter,
writeFilter,

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
index 3da08f5..9dd1a29 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
@@ -144,7 +144,7 @@ public class BaseCache implements Cache {
 
   @Override
   public void put(CacheKey key, CacheValue value) {
-    addToReleaseQueue(_cacheMap.put(key, value));
+    _cacheMap.put(key, value);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
index d8f9a97..0313fa6 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
@@ -38,7 +38,8 @@ public class CacheIndexInput extends IndexInput {
   private long _position;
   private int _blockPosition;
 
-  public CacheIndexInput(CacheDirectory directory, String fileName, IndexInput indexInput,
Cache cache) throws IOException {
+  public CacheIndexInput(CacheDirectory directory, String fileName, IndexInput indexInput,
Cache cache)
+      throws IOException {
     super(fileName);
     _directory = directory;
     _fileName = fileName;
@@ -98,6 +99,8 @@ public class CacheIndexInput extends IndexInput {
   }
 
   private void fill() throws IOException {
+//    System.out.println("Filling [" + _directory + "/" + _fileName + "] @ [" + getBlockId()
+ "] length [" + _fileLength
+//        + "]");
     _key.setBlockId(getBlockId());
     _cacheValue = _cache.get(_key);
     if (_cacheValue == null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
index 0ebc43e..9e33986 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
@@ -25,16 +25,16 @@ import org.apache.lucene.store.IndexOutput;
 public class CacheIndexOutput extends IndexOutput {
 
   private final IndexOutput _indexOutput;
-  private final int _fileBufferSize;
   private final Cache _cache;
   private final String _fileName;
   private final CacheDirectory _directory;
   private final long _fileId;
+  private final int _fileBufferSize;
+  private final int _cacheBlockSize;
 
   private long _position;
   private byte[] _buffer;
   private int _bufferPosition;
-  private int _cacheBlockSize;
 
   public CacheIndexOutput(CacheDirectory directory, String fileName, IndexOutput indexOutput,
Cache cache)
       throws IOException {
@@ -49,6 +49,11 @@ public class CacheIndexOutput extends IndexOutput {
   }
 
   @Override
+  public void setLength(long length) throws IOException {
+    
+  }
+
+  @Override
   public void writeByte(byte b) throws IOException {
     tryToFlush();
     _buffer[_bufferPosition] = b;
@@ -81,23 +86,26 @@ public class CacheIndexOutput extends IndexOutput {
   }
 
   private void flushInternal() throws IOException {
-    CacheValue cacheValue = _cache.newInstance(_directory, _fileName);
-    int length = _cacheBlockSize - (_cacheBlockSize - _bufferPosition);
-    int l = length;
-    int o = 0;
-    while (l > 0) {
-      int il = Math.min(_fileBufferSize, l);
-      _indexOutput.writeBytes(_buffer, o, il);
-      o += il;
-      l -= il;
+    int length = _cacheBlockSize - remaining();
+    if (length == 0) {
+      return;
     }
+    CacheValue cacheValue = _cache.newInstance(_directory, _fileName);
+    writeBufferToOutputStream(length);
     cacheValue.write(0, _buffer, 0, length);
-    _cache.put(new CacheKey(_fileId, getBlockId()), cacheValue);
+    long blockId = (_position - length) / _cacheBlockSize;
+    _cache.put(new CacheKey(_fileId, blockId), cacheValue);
     _bufferPosition = 0;
   }
 
-  private long getBlockId() {
-    return _position / _cacheBlockSize;
+  private void writeBufferToOutputStream(int len) throws IOException {
+    int offset = 0;
+    while (len > 0) {
+      int length = Math.min(_fileBufferSize, len);
+      _indexOutput.writeBytes(_buffer, offset, length);
+      len -= length;
+      offset += length;
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
index 96e0d8c..46af604 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
@@ -62,9 +62,9 @@ public class UnsafeCacheValue extends BaseCacheValue {
   }
 
   private int getCapacity(int length) {
-    if (length < MINIMUM_SIZE) {
-      return MINIMUM_SIZE;
-    }
+//    if (length < MINIMUM_SIZE) {
+//      return MINIMUM_SIZE;
+//    }
     return length;
   }
 
@@ -94,6 +94,7 @@ public class UnsafeCacheValue extends BaseCacheValue {
       _released = true;
     } else {
       System.out.println("released twice?");
+      new Throwable().printStackTrace();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/562537d1/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
index 7cc79a3..8e3a285 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
@@ -35,10 +35,13 @@ public class CacheDirectoryTest {
 
   @Before
   public void setup() {
+    int totalNumberOfBytes = 1000000;
+    int fileBufferSize = 127;
+    final int blockSize = 131;
     FileNameBlockSize fileNameBlockSize = new FileNameBlockSize() {
       @Override
       public int getBlockSize(String directoryName, String fileName) {
-        return 131;
+        return blockSize;
       }
     };
     FileNameFilter writeFilter = new FileNameFilter() {
@@ -53,7 +56,8 @@ public class CacheDirectoryTest {
         return true;
       }
     };
-    Cache cache = new BaseCache(10000, 127, fileNameBlockSize, readFilter, writeFilter, STORE.ON_HEAP);
+    Cache cache = new BaseCache(totalNumberOfBytes, fileBufferSize, fileNameBlockSize, readFilter,
writeFilter,
+        STORE.ON_HEAP);
     Directory directory = newDirectory();
     BufferStore.init(128, 128);
     _cacheDirectory = new CacheDirectory("test", directory, cache);
@@ -102,24 +106,31 @@ public class CacheDirectoryTest {
 
   @Test
   public void test3() throws IOException, InterruptedException {
-    Thread.sleep(30000);
+    // Thread.sleep(30000);
     IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
     IndexWriter writer = new IndexWriter(_cacheDirectory, conf);
-    int docs = 10000;
+    int docs = 100000;
     for (int i = 0; i < docs; i++) {
+      if (i % 500 == 0) {
+        System.out.println(i);
+      }
       writer.addDocument(newDoc());
-      Thread.sleep(1);
+      // Thread.sleep(1);
     }
     writer.close();
+    System.out.println("done writing");
 
     DirectoryReader reader = DirectoryReader.open(_cacheDirectory);
+    System.out.println("done opening");
     assertEquals(docs, reader.numDocs());
 
     Document document = reader.document(0);
+    System.out.println("done fetching");
     System.out.println(document);
 
     IndexSearcher searcher = new IndexSearcher(reader);
     TopDocs topDocs = searcher.search(new TermQuery(new Term("test", "test")), 10);
+    System.out.println("done searching");
     assertEquals(docs, topDocs.totalHits);
 
     reader.close();


Mime
View raw message