incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/8] git commit: Blur store and pom files have been updated, everything else is still broken.
Date Tue, 16 Oct 2012 00:57:06 GMT
Blur store and pom files have been updated, everything else is still broken.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/39539e56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/39539e56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/39539e56

Branch: refs/heads/lucene-4.0.0
Commit: 39539e56d65cfb75d45b9c1000022f2b7f921dd3
Parents: 788e9e9
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Oct 13 14:29:57 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Oct 13 14:29:57 2012 -0400

----------------------------------------------------------------------
 src/blur-core/pom.xml                              |   18 +-
 src/blur-demo/pom.xml                              |    4 +-
 src/blur-gui/pom.xml                               |    8 +-
 src/blur-jdbc/pom.xml                              |    6 +-
 src/blur-mapred/pom.xml                            |   10 +-
 src/blur-store/pom.xml                             |   12 +-
 .../apache/blur/index/OptimisticLockFactory.java   |  102 ++
 .../java/org/apache/blur/lucene/EscapeRewrite.java |  334 ------
 .../org/apache/blur/lucene/LuceneConstant.java     |   25 -
 .../apache/blur/lucene/LuceneVersionConstant.java  |   28 +
 .../java/org/apache/blur/store/BufferStore.java    |  113 --
 .../blur/store/CustomBufferedIndexInput.java       |  277 -----
 .../apache/blur/store/blockcache/BlockCache.java   |   28 +-
 .../blur/store/blockcache/BlockDirectory.java      |   85 +-
 .../blur/store/blockcache/BlockDirectoryCache.java |   11 +-
 .../org/apache/blur/store/buffer/BufferStore.java  |  116 ++
 .../store/buffer/ReusedBufferedIndexInput.java     |  361 +++++++
 .../compressed/CompressedFieldDataDirectory.java   |  813 ---------------
 .../org/apache/blur/store/hdfs/ChangeFileExt.java  |   38 -
 .../apache/blur/store/hdfs/ConvertDirectory.java   |   62 --
 .../apache/blur/store/hdfs/CopyFromHdfsLocal.java  |   48 -
 .../org/apache/blur/store/hdfs/HdfsDirectory.java  |  432 +++------
 .../org/apache/blur/store/hdfs/HdfsFileReader.java |  188 ----
 .../org/apache/blur/store/hdfs/HdfsFileWriter.java |   99 --
 .../org/apache/blur/store/hdfs/HdfsMetaBlock.java  |   61 --
 .../apache/blur/store/hdfs/NullIndexOutput.java    |   70 --
 .../apache/blur/store/lock/BlurLockFactory.java    |  102 --
 .../apache/lucene/index/WarmUpByFieldBounds.java   |  217 ----
 .../lucene/index/WarmUpByFieldBoundsStatus.java    |   25 -
 .../org/apache/blur/store/BenchmarkDirectory.java  |  148 ---
 .../apache/blur/store/BenchmarkDirectoryNrt.java   |  160 ---
 .../org/apache/blur/store/HdfsDirectoryTest.java   |   26 +-
 .../java/org/apache/blur/store/UsingHdfsDir.java   |  112 --
 .../blur/store/blockcache/BlockCacheTest.java      |   15 +-
 .../blur/store/blockcache/BlockDirectoryTest.java  |   12 +-
 .../CompressedFieldDataDirectoryTest.java          |  143 ---
 src/blur-testsuite/pom.xml                         |   14 +-
 src/blur-thrift/pom.xml                            |    6 +-
 src/blur-util/pom.xml                              |   10 +-
 src/pom.xml                                        |    2 +-
 40 files changed, 849 insertions(+), 3492 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-core/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-core/pom.xml b/src/blur-core/pom.xml
index 0294a79..dc2ad36 100644
--- a/src/blur-core/pom.xml
+++ b/src/blur-core/pom.xml
@@ -22,7 +22,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
-		<version>0.1.3</version>
+		<version>0.2.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.apache.blur</groupId>
@@ -38,28 +38,28 @@ under the License.
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-thrift</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-store</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-util</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-gui</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 			<type>jar</type>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-gui</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 			<type>war</type>
 		</dependency>
 		<dependency>
@@ -71,7 +71,7 @@ under the License.
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-test</artifactId>
-			<version>0.20.2-cdh3u5</version>
+			<version>1.0.3</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>
@@ -105,10 +105,6 @@ under the License.
 			<id>libdir</id>
 			<url>file://${basedir}/../lib</url>
 		</repository>
-		<repository>
-			<id>cloudera</id>
-			<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
-		</repository>
 	</repositories>
 
 	<build>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-demo/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-demo/pom.xml b/src/blur-demo/pom.xml
index 6cb2a40..4051dea 100644
--- a/src/blur-demo/pom.xml
+++ b/src/blur-demo/pom.xml
@@ -22,7 +22,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
-		<version>0.1.3</version>
+		<version>0.2.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.apache.blur</groupId>
@@ -34,7 +34,7 @@ under the License.
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-core</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>log4j</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-gui/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-gui/pom.xml b/src/blur-gui/pom.xml
index 9529928..46629ca 100644
--- a/src/blur-gui/pom.xml
+++ b/src/blur-gui/pom.xml
@@ -22,7 +22,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
-		<version>0.1.3</version>
+		<version>0.2.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.apache.blur</groupId>
@@ -38,17 +38,17 @@ under the License.
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-thrift</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-store</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-util</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>log4j</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-jdbc/pom.xml b/src/blur-jdbc/pom.xml
index 77ec480..db5bff0 100644
--- a/src/blur-jdbc/pom.xml
+++ b/src/blur-jdbc/pom.xml
@@ -22,7 +22,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
-		<version>0.1.3</version>
+		<version>0.2.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.apache.blur</groupId>
@@ -34,12 +34,12 @@ under the License.
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-thrift</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-util</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>log4j</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-mapred/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-mapred/pom.xml b/src/blur-mapred/pom.xml
index 81546c3..e2f0c8a 100644
--- a/src/blur-mapred/pom.xml
+++ b/src/blur-mapred/pom.xml
@@ -14,7 +14,7 @@
 	<parent>
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
-		<version>0.1.3</version>
+		<version>0.2.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.apache.blur</groupId>
@@ -26,23 +26,23 @@
 		<dependency>
 			<groupId>org.apache.zookeeper</groupId>
 			<artifactId>zookeeper</artifactId>
-			<version>3.3.4</version>
+			<version>3.4.4</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-core</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-store</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-util</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>log4j</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-store/pom.xml b/src/blur-store/pom.xml
index 2c53ec1..8b13cda 100644
--- a/src/blur-store/pom.xml
+++ b/src/blur-store/pom.xml
@@ -22,7 +22,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
-		<version>0.1.3</version>
+		<version>0.2.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.apache.blur</groupId>
@@ -34,12 +34,12 @@ under the License.
 		<dependency>
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-util</artifactId>
-			<version>0.1.3</version>
+			<version>0.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.lucene</groupId>
 			<artifactId>lucene-core</artifactId>
-			<version>3.6.1.lm</version>
+			<version>4.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>commons-cli</groupId>
@@ -50,7 +50,7 @@ under the License.
 		<dependency>
 			<groupId>com.googlecode.concurrentlinkedhashmap</groupId>
 			<artifactId>concurrentlinkedhashmap-lru</artifactId>
-			<version>1.2</version>
+			<version>1.3.1</version>
 		</dependency>
 		<dependency>
 			<groupId>log4j</groupId>
@@ -83,10 +83,6 @@ under the License.
 			<id>libdir</id>
 			<url>file://${basedir}/../lib</url>
 		</repository>
-		<repository>
-			<id>cloudera</id>
-			<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
-		</repository>
 	</repositories>
 
 	<build>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java b/src/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
new file mode 100644
index 0000000..f6347bc
--- /dev/null
+++ b/src/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
@@ -0,0 +1,102 @@
+package org.apache.blur.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+
+public class OptimisticLockFactory extends LockFactory {
+
+  private final Configuration _configuration;
+  private final FileSystem _fileSystem;
+  private final String _baseLockKey;
+  private byte[] _lockKey;
+  private final Path _dir;
+
+  public OptimisticLockFactory(Configuration configuration, Path dir, String host, int pid) throws IOException {
+    _configuration = configuration;
+    _dir = dir;
+    _fileSystem = _dir.getFileSystem(_configuration);
+    _baseLockKey = host + "/" + pid;
+  }
+
+  @Override
+  public Lock makeLock(String lockName) {
+    final Path lockPath = new Path(_dir, lockName);
+    return new Lock() {
+      private boolean _set;
+
+      @Override
+      public boolean obtain() throws IOException {
+        if (_set) {
+          throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
+        }
+        try {
+          _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
+          FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
+          outputStream.write(_lockKey);
+          outputStream.close();
+        } finally {
+          _set = true;
+        }
+        return true;
+      }
+
+      @Override
+      public void release() throws IOException {
+        _fileSystem.delete(lockPath, false);
+      }
+
+      @Override
+      public boolean isLocked() throws IOException {
+        if (!_set) {
+          return false;
+        }
+        if (!_fileSystem.exists(lockPath)) {
+          return false;
+        }
+        FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
+        long len = fileStatus.getLen();
+        if (len != _lockKey.length) {
+          return false;
+        }
+        byte[] buf = new byte[_lockKey.length];
+        FSDataInputStream inputStream = _fileSystem.open(lockPath);
+        inputStream.readFully(buf);
+        inputStream.close();
+        if (Arrays.equals(_lockKey, buf)) {
+          return true;
+        }
+        return false;
+      }
+    };
+  }
+
+  @Override
+  public void clearLock(String lockName) throws IOException {
+    _fileSystem.delete(new Path(_dir, lockName), false);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/lucene/EscapeRewrite.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/EscapeRewrite.java b/src/blur-store/src/main/java/org/apache/blur/lucene/EscapeRewrite.java
deleted file mode 100644
index b1e0c7b..0000000
--- a/src/blur-store/src/main/java/org/apache/blur/lucene/EscapeRewrite.java
+++ /dev/null
@@ -1,334 +0,0 @@
-package org.apache.blur.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.TermFreqVector;
-import org.apache.lucene.index.TermPositions;
-import org.apache.lucene.index.TermVectorMapper;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-
-@SuppressWarnings("deprecation")
-public class EscapeRewrite {
-
-  public static void main(String[] args) throws CorruptIndexException, IOException {
-    Directory directory = FSDirectory.open(new File("/Users/amccurry/Documents/workspace/low-lat-lucene-rt/index"));
-    AtomicBoolean running = new AtomicBoolean(true);
-    IndexReader indexReader = IndexReader.open(directory);
-    // IndexReader reader = indexReader;
-    IndexReader reader = wrap(indexReader, running);
-    Query query = new WildcardQuery(new Term("id", "*0*"));
-    // Query query = new TermQuery(new Term("id","0"));
-    escapeIn(running, TimeUnit.SECONDS.toMillis(5));
-    IndexSearcher searcher = new IndexSearcher(reader);
-    long s1 = System.nanoTime();
-    Query rewrite = searcher.rewrite(query);
-    long e1 = System.nanoTime();
-
-    long s2 = System.nanoTime();
-    TopDocs topDocs = searcher.search(rewrite, 10);
-    long e2 = System.nanoTime();
-
-    System.out.println((e1 - s1) / 1000000.0 + " " + rewrite);
-    System.out.println((e2 - s2) / 1000000.0 + " " + topDocs.totalHits);
-  }
-
-  private static void escapeIn(final AtomicBoolean running, final long millis) {
-    new Thread(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          Thread.sleep(millis);
-        } catch (InterruptedException e) {
-          e.printStackTrace();
-        }
-        running.set(false);
-      }
-    }).start();
-  }
-
-  public static IndexReader wrap(IndexReader reader, AtomicBoolean running) {
-    return new IndexReaderRewriteEscape(reader, running);
-  }
-
-  public static class IndexReaderRewriteEscape extends IndexReader {
-    private IndexReader reader;
-    private AtomicBoolean running;
-
-    public IndexReaderRewriteEscape(IndexReader reader, AtomicBoolean running) {
-      this.reader = reader;
-      this.running = running;
-    }
-
-    public String toString() {
-      return reader.toString();
-    }
-
-    public IndexReader reopen() throws CorruptIndexException, IOException {
-      return reader.reopen();
-    }
-
-    public IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
-      return reader.reopen(openReadOnly);
-    }
-
-    public IndexReader reopen(IndexCommit commit) throws CorruptIndexException, IOException {
-      return reader.reopen(commit);
-    }
-
-    public IndexReader reopen(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
-      return reader.reopen(writer, applyAllDeletes);
-    }
-
-    public Directory directory() {
-      return reader.directory();
-    }
-
-    public long getVersion() {
-      return reader.getVersion();
-    }
-
-    public boolean isOptimized() {
-      return reader.isOptimized();
-    }
-
-    public TermFreqVector[] getTermFreqVectors(int docNumber) throws IOException {
-      return reader.getTermFreqVectors(docNumber);
-    }
-
-    public TermFreqVector getTermFreqVector(int docNumber, String field) throws IOException {
-      return reader.getTermFreqVector(docNumber, field);
-    }
-
-    public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException {
-      reader.getTermFreqVector(docNumber, field, mapper);
-    }
-
-    public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
-      reader.getTermFreqVector(docNumber, mapper);
-    }
-
-    public int numDocs() {
-      return reader.numDocs();
-    }
-
-    public int maxDoc() {
-      return reader.maxDoc();
-    }
-
-    public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
-      return reader.document(n, fieldSelector);
-    }
-
-    public boolean hasDeletions() {
-      return reader.hasDeletions();
-    }
-
-    public boolean hasNorms(String field) throws IOException {
-      return reader.hasNorms(field);
-    }
-
-    public int docFreq(Term t) throws IOException {
-      return reader.docFreq(t);
-    }
-
-    public boolean equals(Object arg0) {
-      return reader.equals(arg0);
-    }
-
-    public Map<String, String> getCommitUserData() {
-      return reader.getCommitUserData();
-    }
-
-    public FieldInfos getFieldInfos() {
-      return reader.getFieldInfos();
-    }
-
-    public IndexCommit getIndexCommit() throws IOException {
-      return reader.getIndexCommit();
-    }
-
-    public Object getCoreCacheKey() {
-      return reader.getCoreCacheKey();
-    }
-
-    public Object getDeletesCacheKey() {
-      return reader.getDeletesCacheKey();
-    }
-
-    public long getUniqueTermCount() throws IOException {
-      return reader.getUniqueTermCount();
-    }
-
-    public int getTermInfosIndexDivisor() {
-      return reader.getTermInfosIndexDivisor();
-    }
-
-    public int hashCode() {
-      return reader.hashCode();
-    }
-
-    public boolean isCurrent() throws CorruptIndexException, IOException {
-      return reader.isCurrent();
-    }
-
-    public boolean isDeleted(int n) {
-      return reader.isDeleted(n);
-    }
-
-    public byte[] norms(String field) throws IOException {
-      return reader.norms(field);
-    }
-
-    public void norms(String field, byte[] bytes, int offset) throws IOException {
-      reader.norms(field, bytes, offset);
-    }
-
-    public TermDocs termDocs(Term term) throws IOException {
-      return reader.termDocs(term);
-    }
-
-    public TermDocs termDocs() throws IOException {
-      return reader.termDocs();
-    }
-
-    public TermPositions termPositions() throws IOException {
-      return reader.termPositions();
-    }
-
-    public Object clone() {
-      IndexReaderRewriteEscape clone = (IndexReaderRewriteEscape) super.clone();
-      clone.reader = (IndexReader) reader.clone();
-      return clone;
-    }
-
-    public IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
-      IndexReaderRewriteEscape clone = (IndexReaderRewriteEscape) super.clone();
-      clone.reader = reader.clone(openReadOnly);
-      return clone;
-    }
-
-    public IndexReader[] getSequentialSubReaders() {
-      return wrap(reader.getSequentialSubReaders(), running);
-    }
-
-    public TermEnum terms() throws IOException {
-      return wrap(reader.terms(), running);
-    }
-
-    public TermEnum terms(Term t) throws IOException {
-      return wrap(reader.terms(t), running);
-    }
-
-    @Override
-    protected void doSetNorm(int doc, String field, byte value) throws CorruptIndexException, IOException {
-      reader.setNorm(doc, field, value);
-    }
-
-    @Override
-    protected void doDelete(int docNum) throws CorruptIndexException, IOException {
-      reader.deleteDocument(docNum);
-    }
-
-    @Override
-    protected void doUndeleteAll() throws CorruptIndexException, IOException {
-      reader.undeleteAll();
-    }
-
-    @Override
-    protected void doCommit(Map<String, String> commitUserData) throws IOException {
-      reader.commit(commitUserData);
-    }
-
-    @Override
-    protected void doClose() throws IOException {
-      reader.close();
-    }
-  }
-
-  public static TermEnum wrap(final TermEnum terms, final AtomicBoolean running) {
-    return new TermEnum() {
-
-      private int count = 0;
-      private boolean quit = false;
-
-      @Override
-      public Term term() {
-        Term term = terms.term();
-        System.out.println(term);
-        return term;
-      }
-
-      @Override
-      public boolean next() throws IOException {
-        if (quit) {
-          return false;
-        }
-        if (count >= 10000) {
-          if (!running.get()) {
-            quit = true;
-          }
-          count = 0;
-        }
-        count++;
-        return terms.next();
-      }
-
-      @Override
-      public int docFreq() {
-        return terms.docFreq();
-      }
-
-      @Override
-      public void close() throws IOException {
-        terms.close();
-      }
-    };
-  }
-
-  public static IndexReader[] wrap(IndexReader[] sequentialSubReaders, AtomicBoolean running) {
-    if (sequentialSubReaders == null) {
-      return null;
-    }
-    IndexReader[] result = new IndexReader[sequentialSubReaders.length];
-    for (int i = 0; i < sequentialSubReaders.length; i++) {
-      result[i] = wrap(sequentialSubReaders[i], running);
-    }
-    return result;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneConstant.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneConstant.java b/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneConstant.java
deleted file mode 100644
index b4df0c4..0000000
--- a/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneConstant.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.apache.blur.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.lucene.util.Version;
-
-public class LuceneConstant {
-
-  public static final Version LUCENE_VERSION = Version.LUCENE_36;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java b/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
new file mode 100644
index 0000000..a6b33e0
--- /dev/null
+++ b/src/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
@@ -0,0 +1,28 @@
+package org.apache.blur.lucene;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.lucene.util.Version;
+
+public class LuceneVersionConstant {
+
+  /**
+   * All Lucene Version references should refer to this constant.
+   */
+  public static final Version LUCENE_VERSION = Version.LUCENE_40;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/BufferStore.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/BufferStore.java b/src/blur-store/src/main/java/org/apache/blur/store/BufferStore.java
deleted file mode 100644
index dbd9fcd..0000000
--- a/src/blur-store/src/main/java/org/apache/blur/store/BufferStore.java
+++ /dev/null
@@ -1,113 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.blur.BlurConfiguration;
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.blur.metrics.BlurMetrics;
-
-
-public class BufferStore {
-
-  private static final Log LOG = LogFactory.getLog(BufferStore.class);
-
-  private static BlockingQueue<byte[]> _1024 = setupBuffers(1024, 1);
-  private static BlockingQueue<byte[]> _8192 = setupBuffers(8192, 1);
-  public static AtomicLong _blurShardBuffercacheLost = new AtomicLong();
-  public static AtomicLong _blurShardBuffercacheAllocate1024 = new AtomicLong();
-  public static AtomicLong _blurShardBuffercacheAllocate8192 = new AtomicLong();
-  public static AtomicLong _blurShardBuffercacheAllocateOther = new AtomicLong();
-
-  public static void init(BlurConfiguration configuration, BlurMetrics metrics) {
-    int _1024Size = configuration.getInt("blur.shard.buffercache.1024", 8192);
-    int _8192Size = configuration.getInt("blur.shard.buffercache.8192", 8192);
-    LOG.info("Initializing the 1024 buffers with [{0}] buffers.", _1024Size);
-    _1024 = setupBuffers(1024, _1024Size);
-    LOG.info("Initializing the 8192 buffers with [{0}] buffers.", _8192Size);
-    _8192 = setupBuffers(8192, _8192Size);
-    _blurShardBuffercacheLost = metrics.blurShardBuffercacheLost;
-    _blurShardBuffercacheAllocate1024 = metrics.blurShardBuffercacheAllocate1024;
-    _blurShardBuffercacheAllocate8192 = metrics.blurShardBuffercacheAllocate8192;
-    _blurShardBuffercacheAllocateOther = metrics.blurShardBuffercacheAllocateOther;
-  }
-
-  private static BlockingQueue<byte[]> setupBuffers(int bufferSize, int count) {
-    BlockingQueue<byte[]> queue = new ArrayBlockingQueue<byte[]>(count);
-    for (int i = 0; i < count; i++) {
-      queue.add(new byte[bufferSize]);
-    }
-    return queue;
-  }
-
-  public static byte[] takeBuffer(int bufferSize) {
-    switch (bufferSize) {
-    case 1024:
-      return newBuffer1024(_1024.poll());
-    case 8192:
-      return newBuffer8192(_8192.poll());
-    default:
-      return newBuffer(bufferSize);
-    }
-  }
-
-  public static void putBuffer(byte[] buffer) {
-    if (buffer == null) {
-      return;
-    }
-    int bufferSize = buffer.length;
-    switch (bufferSize) {
-    case 1024:
-      checkReturn(_1024.offer(buffer));
-      return;
-    case 8192:
-      checkReturn(_8192.offer(buffer));
-      return;
-    }
-  }
-
-  private static void checkReturn(boolean offer) {
-    if (!offer) {
-      _blurShardBuffercacheLost.incrementAndGet();
-    }
-  }
-
-  private static byte[] newBuffer1024(byte[] buf) {
-    if (buf != null) {
-      return buf;
-    }
-    _blurShardBuffercacheAllocate1024.incrementAndGet();
-    return new byte[1024];
-  }
-
-  private static byte[] newBuffer8192(byte[] buf) {
-    if (buf != null) {
-      return buf;
-    }
-    _blurShardBuffercacheAllocate8192.incrementAndGet();
-    return new byte[8192];
-  }
-
-  private static byte[] newBuffer(int size) {
-    _blurShardBuffercacheAllocateOther.incrementAndGet();
-    return new byte[size];
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/CustomBufferedIndexInput.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/CustomBufferedIndexInput.java b/src/blur-store/src/main/java/org/apache/blur/store/CustomBufferedIndexInput.java
deleted file mode 100644
index b206ec9..0000000
--- a/src/blur-store/src/main/java/org/apache/blur/store/CustomBufferedIndexInput.java
+++ /dev/null
@@ -1,277 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-
-public abstract class CustomBufferedIndexInput extends IndexInput {
-
-  public static final int BUFFER_SIZE = 1024;
-
-  private int bufferSize = BUFFER_SIZE;
-
-  protected byte[] buffer;
-
-  private long bufferStart = 0; // position in file of buffer
-  private int bufferLength = 0; // end of valid bytes
-  private int bufferPosition = 0; // next byte to read
-
-  @Override
-  public byte readByte() throws IOException {
-    if (bufferPosition >= bufferLength)
-      refill();
-    return buffer[bufferPosition++];
-  }
-
-  public CustomBufferedIndexInput(String resourceDesc) {
-    this(resourceDesc, BUFFER_SIZE);
-  }
-
-  public CustomBufferedIndexInput(String resourceDesc, int bufferSize) {
-    super(resourceDesc);
-    checkBufferSize(bufferSize);
-    this.bufferSize = bufferSize;
-  }
-
-  private void checkBufferSize(int bufferSize) {
-    if (bufferSize <= 0)
-      throw new IllegalArgumentException("bufferSize must be greater than 0 (got " + bufferSize + ")");
-  }
-
-  @Override
-  public void readBytes(byte[] b, int offset, int len) throws IOException {
-    readBytes(b, offset, len, true);
-  }
-
-  @Override
-  public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
-
-    if (len <= (bufferLength - bufferPosition)) {
-      // the buffer contains enough data to satisfy this request
-      if (len > 0) // to allow b to be null if len is 0...
-        System.arraycopy(buffer, bufferPosition, b, offset, len);
-      bufferPosition += len;
-    } else {
-      // the buffer does not have enough data. First serve all we've got.
-      int available = bufferLength - bufferPosition;
-      if (available > 0) {
-        System.arraycopy(buffer, bufferPosition, b, offset, available);
-        offset += available;
-        len -= available;
-        bufferPosition += available;
-      }
-      // and now, read the remaining 'len' bytes:
-      if (useBuffer && len < bufferSize) {
-        // If the amount left to read is small enough, and
-        // we are allowed to use our buffer, do it in the usual
-        // buffered way: fill the buffer and copy from it:
-        refill();
-        if (bufferLength < len) {
-          // Throw an exception when refill() could not read len bytes:
-          System.arraycopy(buffer, 0, b, offset, bufferLength);
-          throw new IOException("read past EOF");
-        } else {
-          System.arraycopy(buffer, 0, b, offset, len);
-          bufferPosition = len;
-        }
-      } else {
-        // The amount left to read is larger than the buffer
-        // or we've been asked to not use our buffer -
-        // there's no performance reason not to read it all
-        // at once. Note that unlike the previous code of
-        // this function, there is no need to do a seek
-        // here, because there's no need to reread what we
-        // had in the buffer.
-        long after = bufferStart + bufferPosition + len;
-        if (after > length())
-          throw new IOException("read past EOF");
-        readInternal(b, offset, len);
-        bufferStart = after;
-        bufferPosition = 0;
-        bufferLength = 0; // trigger refill() on read
-      }
-    }
-  }
-
-  @Override
-  public int readInt() throws IOException {
-    if (4 <= (bufferLength - bufferPosition)) {
-      return ((buffer[bufferPosition++] & 0xFF) << 24) | ((buffer[bufferPosition++] & 0xFF) << 16) | ((buffer[bufferPosition++] & 0xFF) << 8) | (buffer[bufferPosition++] & 0xFF);
-    } else {
-      return super.readInt();
-    }
-  }
-
-  @Override
-  public long readLong() throws IOException {
-    if (8 <= (bufferLength - bufferPosition)) {
-      final int i1 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16) | ((buffer[bufferPosition++] & 0xff) << 8)
-          | (buffer[bufferPosition++] & 0xff);
-      final int i2 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16) | ((buffer[bufferPosition++] & 0xff) << 8)
-          | (buffer[bufferPosition++] & 0xff);
-      return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
-    } else {
-      return super.readLong();
-    }
-  }
-
-  @Override
-  public int readVInt() throws IOException {
-    if (5 <= (bufferLength - bufferPosition)) {
-      byte b = buffer[bufferPosition++];
-      int i = b & 0x7F;
-      for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-        b = buffer[bufferPosition++];
-        i |= (b & 0x7F) << shift;
-      }
-      return i;
-    } else {
-      return super.readVInt();
-    }
-  }
-
-  @Override
-  public long readVLong() throws IOException {
-    if (9 <= bufferLength - bufferPosition) {
-      byte b = buffer[bufferPosition++];
-      long i = b & 0x7F;
-      for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-        b = buffer[bufferPosition++];
-        i |= (b & 0x7FL) << shift;
-      }
-      return i;
-    } else {
-      return super.readVLong();
-    }
-  }
-
-  private void refill() throws IOException {
-    long start = bufferStart + bufferPosition;
-    long end = start + bufferSize;
-    if (end > length()) // don't read past EOF
-      end = length();
-    int newLength = (int) (end - start);
-    if (newLength <= 0)
-      throw new IOException("read past EOF");
-
-    if (buffer == null) {
-      buffer = BufferStore.takeBuffer(bufferSize);
-      seekInternal(bufferStart);
-    }
-    readInternal(buffer, 0, newLength);
-    bufferLength = newLength;
-    bufferStart = start;
-    bufferPosition = 0;
-  }
-
-  @Override
-  public final void close() throws IOException {
-    closeInternal();
-    BufferStore.putBuffer(buffer);
-    buffer = null;
-  }
-
-  protected abstract void closeInternal() throws IOException;
-
-  /**
-   * Expert: implements buffer refill. Reads bytes from the current position in
-   * the input.
-   * 
-   * @param b
-   *          the array to read bytes into
-   * @param offset
-   *          the offset in the array to start storing bytes
-   * @param length
-   *          the number of bytes to read
-   */
-  protected abstract void readInternal(byte[] b, int offset, int length) throws IOException;
-
-  @Override
-  public long getFilePointer() {
-    return bufferStart + bufferPosition;
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    if (pos >= bufferStart && pos < (bufferStart + bufferLength))
-      bufferPosition = (int) (pos - bufferStart); // seek within buffer
-    else {
-      bufferStart = pos;
-      bufferPosition = 0;
-      bufferLength = 0; // trigger refill() on read()
-      seekInternal(pos);
-    }
-  }
-
-  /**
-   * Expert: implements seek. Sets current position in this file, where the next
-   * {@link #readInternal(byte[],int,int)} will occur.
-   * 
-   * @see #readInternal(byte[],int,int)
-   */
-  protected abstract void seekInternal(long pos) throws IOException;
-
-  @Override
-  public Object clone() {
-    CustomBufferedIndexInput clone = (CustomBufferedIndexInput) super.clone();
-
-    clone.buffer = null;
-    clone.bufferLength = 0;
-    clone.bufferPosition = 0;
-    clone.bufferStart = getFilePointer();
-
-    return clone;
-  }
-
-  /**
-   * Flushes the in-memory bufer to the given output, copying at most
-   * <code>numBytes</code>.
-   * <p>
-   * <b>NOTE:</b> this method does not refill the buffer, however it does
-   * advance the buffer position.
-   * 
-   * @return the number of bytes actually flushed from the in-memory buffer.
-   */
-  protected int flushBuffer(IndexOutput out, long numBytes) throws IOException {
-    int toCopy = bufferLength - bufferPosition;
-    if (toCopy > numBytes) {
-      toCopy = (int) numBytes;
-    }
-    if (toCopy > 0) {
-      out.writeBytes(buffer, bufferPosition, toCopy);
-      bufferPosition += toCopy;
-    }
-    return toCopy;
-  }
-
-  @Override
-  public void copyBytes(IndexOutput out, long numBytes) throws IOException {
-    assert numBytes >= 0 : "numBytes=" + numBytes;
-
-    while (numBytes > 0) {
-      if (bufferLength == bufferPosition) {
-        refill();
-      }
-      numBytes -= flushBuffer(out, numBytes);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockCache.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockCache.java b/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockCache.java
index 02c9dbf..0358fc8 100644
--- a/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockCache.java
+++ b/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockCache.java
@@ -20,35 +20,27 @@ import java.nio.ByteBuffer;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.blur.metrics.BlurMetrics;
-
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
 import com.googlecode.concurrentlinkedhashmap.EvictionListener;
 
 public class BlockCache {
 
   public static final int _128M = 134217728;
-  public static final int _32K = 32768;
+  public static final int _8K = 8192;
   private final ConcurrentMap<BlockCacheKey, BlockCacheLocation> _cache;
   private final ByteBuffer[] _slabs;
   private final BlockLocks[] _locks;
   private final AtomicInteger[] _lockCounters;
-  private final int _blockSize;
+  private final int _blockSize = _8K;
   private final int _numberOfBlocksPerSlab;
   private final int _maxEntries;
-  private final BlurMetrics _metrics;
-
-  public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory) {
-    this(metrics, directAllocation, totalMemory, _128M);
-  }
 
-  public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory, int slabSize) {
-    this(metrics, directAllocation, totalMemory, slabSize, _32K);
+  public BlockCache(boolean directAllocation, long totalMemory) {
+    this(directAllocation, totalMemory, _128M);
   }
 
-  public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory, int slabSize, int blockSize) {
-    _metrics = metrics;
-    _numberOfBlocksPerSlab = slabSize / blockSize;
+  public BlockCache(boolean directAllocation, long totalMemory, int slabSize) {
+    _numberOfBlocksPerSlab = slabSize / _blockSize;
     int numberOfSlabs = (int) (totalMemory / slabSize);
 
     _slabs = new ByteBuffer[numberOfSlabs];
@@ -57,9 +49,9 @@ public class BlockCache {
     _maxEntries = (_numberOfBlocksPerSlab * numberOfSlabs) - 1;
     for (int i = 0; i < numberOfSlabs; i++) {
       if (directAllocation) {
-        _slabs[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerSlab * blockSize);
+        _slabs[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerSlab * _blockSize);
       } else {
-        _slabs[i] = ByteBuffer.allocate(_numberOfBlocksPerSlab * blockSize);
+        _slabs[i] = ByteBuffer.allocate(_numberOfBlocksPerSlab * _blockSize);
       }
       _locks[i] = new BlockLocks(_numberOfBlocksPerSlab);
       _lockCounters[i] = new AtomicInteger();
@@ -72,7 +64,6 @@ public class BlockCache {
       }
     };
     _cache = new ConcurrentLinkedHashMap.Builder<BlockCacheKey, BlockCacheLocation>().maximumWeightedCapacity(_maxEntries).listener(listener).build();
-    _blockSize = blockSize;
   }
 
   private void releaseLocation(BlockCacheLocation location) {
@@ -84,8 +75,6 @@ public class BlockCache {
     location.setRemoved(true);
     _locks[slabId].clear(block);
     _lockCounters[slabId].decrementAndGet();
-    _metrics.blockCacheEviction.incrementAndGet();
-    _metrics.blockCacheSize.decrementAndGet();
   }
 
   public boolean store(BlockCacheKey blockCacheKey, byte[] data) {
@@ -109,7 +98,6 @@ public class BlockCache {
     slab.put(data, 0, _blockSize);
     if (newLocation) {
       releaseLocation(_cache.put(blockCacheKey.clone(), location));
-      _metrics.blockCacheSize.incrementAndGet();
     }
     return true;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java b/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
index fed1039..d9efecf 100644
--- a/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
+++ b/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
@@ -16,19 +16,23 @@ package org.apache.blur.store.blockcache;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Set;
 
-import org.apache.blur.store.BufferStore;
-import org.apache.blur.store.CustomBufferedIndexInput;
+import org.apache.blur.store.buffer.BufferStore;
+import org.apache.blur.store.buffer.ReusedBufferedIndexInput;
+import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockFactory;
 
-
 public class BlockDirectory extends Directory {
 
   public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
@@ -97,14 +101,6 @@ public class BlockDirectory extends Directory {
     setLockFactory(directory.getLockFactory());
   }
 
-  public IndexInput openInput(String name, int bufferSize) throws IOException {
-    final IndexInput source = _directory.openInput(name, _blockSize);
-    if (_blockCacheFileTypes == null || isCachableFile(name)) {
-      return new CachedIndexInput(source, _blockSize, name, getFileCacheName(name), _cache, bufferSize);
-    }
-    return source;
-  }
-
   private boolean isCachableFile(String name) {
     for (String ext : _blockCacheFileTypes) {
       if (name.endsWith(ext)) {
@@ -115,11 +111,15 @@ public class BlockDirectory extends Directory {
   }
 
   @Override
-  public IndexInput openInput(final String name) throws IOException {
-    return openInput(name, _blockSize);
+  public IndexInput openInput(final String name, IOContext context) throws IOException {
+    final IndexInput source = _directory.openInput(name, context);
+    if (_blockCacheFileTypes == null || isCachableFile(name)) {
+      return new CachedIndexInput(source, _blockSize, name, getFileCacheName(name), _cache, context);
+    }
+    return source;
   }
 
-  static class CachedIndexInput extends CustomBufferedIndexInput {
+  static class CachedIndexInput extends ReusedBufferedIndexInput {
 
     private IndexInput _source;
     private int _blockSize;
@@ -127,8 +127,8 @@ public class BlockDirectory extends Directory {
     private String _cacheName;
     private Cache _cache;
 
-    public CachedIndexInput(IndexInput source, int blockSize, String name, String cacheName, Cache cache, int bufferSize) {
-      super(name, bufferSize);
+    public CachedIndexInput(IndexInput source, int blockSize, String name, String cacheName, Cache cache, IOContext context) {
+      super(name, context);
       _source = source;
       _blockSize = blockSize;
       _fileLength = source.length();
@@ -137,7 +137,7 @@ public class BlockDirectory extends Directory {
     }
 
     @Override
-    public Object clone() {
+    public CachedIndexInput clone() {
       CachedIndexInput clone = (CachedIndexInput) super.clone();
       clone._source = (IndexInput) _source.clone();
       return clone;
@@ -208,75 +208,90 @@ public class BlockDirectory extends Directory {
   }
 
   private String getFileCacheName(String name) throws IOException {
-    return _dirName + "/" + name + ":" + fileModified(name);
+    return _dirName + "/" + name + ":" + getFileModified(name);
+  }
+
+  private long getFileModified(String name) throws IOException {
+    if (_directory instanceof FSDirectory) {
+      File directory = ((FSDirectory) _directory).getDirectory();
+      File file = new File(directory,name);
+      if (!file.exists()) {
+        throw new FileNotFoundException("File [" + name + "] not found");
+      }
+      return file.lastModified();
+    } else if (_directory instanceof HdfsDirectory) {
+      return ((HdfsDirectory) _directory).getFileModified(name);
+    } else {
+      throw new RuntimeException("Not supported");
+    }
   }
 
   public void clearLock(String name) throws IOException {
     _directory.clearLock(name);
   }
 
-  public void copy(Directory to, String src, String dest) throws IOException {
-    _directory.copy(to, src, dest);
+  @Override
+  public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
+    _directory.copy(to, src, dest, context);
   }
 
+  @Override
   public LockFactory getLockFactory() {
     return _directory.getLockFactory();
   }
 
+  @Override
   public String getLockID() {
     return _directory.getLockID();
   }
 
+  @Override
   public Lock makeLock(String name) {
     return _directory.makeLock(name);
   }
 
+  @Override
   public void setLockFactory(LockFactory lockFactory) throws IOException {
     _directory.setLockFactory(lockFactory);
   }
 
+  @Override
   public void sync(Collection<String> names) throws IOException {
     _directory.sync(names);
   }
 
-  @SuppressWarnings("deprecation")
-  public void sync(String name) throws IOException {
-    _directory.sync(name);
-  }
-
   public String toString() {
     return _directory.toString();
   }
 
-  public IndexOutput createOutput(String name) throws IOException {
-    return _directory.createOutput(name);
+  @Override
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
+    return _directory.createOutput(name, context);
   }
 
+  @Override
   public void deleteFile(String name) throws IOException {
     _cache.delete(getFileCacheName(name));
     _directory.deleteFile(name);
   }
 
+  @Override
   public boolean fileExists(String name) throws IOException {
     return _directory.fileExists(name);
   }
 
+  @Override
   public long fileLength(String name) throws IOException {
     return _directory.fileLength(name);
   }
 
-  @SuppressWarnings("deprecation")
-  public long fileModified(String name) throws IOException {
-    return _directory.fileModified(name);
-  }
-
+  @Override
   public String[] listAll() throws IOException {
     return _directory.listAll();
   }
 
-  @SuppressWarnings("deprecation")
-  public void touchFile(String name) throws IOException {
-    _directory.touchFile(name);
+  public Directory getDirectory() {
+    return _directory;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectoryCache.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectoryCache.java b/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectoryCache.java
index becd4ba..a7117e3 100644
--- a/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectoryCache.java
+++ b/src/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectoryCache.java
@@ -20,18 +20,14 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.blur.metrics.BlurMetrics;
-
 
 public class BlockDirectoryCache implements Cache {
   private BlockCache _blockCache;
   private AtomicInteger _counter = new AtomicInteger();
   private Map<String, Integer> _names = new ConcurrentHashMap<String, Integer>();
-  private BlurMetrics _blurMetrics;
 
-  public BlockDirectoryCache(BlockCache blockCache, BlurMetrics blurMetrics) {
+  public BlockDirectoryCache(BlockCache blockCache) {
     _blockCache = blockCache;
-    _blurMetrics = blurMetrics;
   }
 
   @Override
@@ -62,11 +58,6 @@ public class BlockDirectoryCache implements Cache {
     blockCacheKey.setBlock(blockId);
     blockCacheKey.setFile(file);
     boolean fetch = _blockCache.fetch(blockCacheKey, b, blockOffset, off, lengthToReadInBlock);
-    if (fetch) {
-      _blurMetrics.blockCacheHit.incrementAndGet();
-    } else {
-      _blurMetrics.blockCacheMiss.incrementAndGet();
-    }
     return fetch;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java b/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java
new file mode 100644
index 0000000..32c1bb2
--- /dev/null
+++ b/src/blur-store/src/main/java/org/apache/blur/store/buffer/BufferStore.java
@@ -0,0 +1,116 @@
+package org.apache.blur.store.buffer;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.metrics.BlurMetrics;
+
+
+public class BufferStore {
+
+  private static final Log LOG = LogFactory.getLog(BufferStore.class);
+
+  private static BlockingQueue<byte[]> _1024 = setupBuffers(1024, 1);
+  private static BlockingQueue<byte[]> _8192 = setupBuffers(8192, 1);
+  
+  // @TODO need to do a performance evaluation on the use of the atomic long counters.
+  
+  public static AtomicLong _blurShardBuffercacheLost = new AtomicLong();
+  public static AtomicLong _blurShardBuffercacheAllocate1024 = new AtomicLong();
+  public static AtomicLong _blurShardBuffercacheAllocate8192 = new AtomicLong();
+  public static AtomicLong _blurShardBuffercacheAllocateOther = new AtomicLong();
+
+  public static void init(BlurConfiguration configuration, BlurMetrics metrics) {
+    int _1024Size = configuration.getInt("blur.shard.buffercache.1024", 8192);
+    int _8192Size = configuration.getInt("blur.shard.buffercache.8192", 8192);
+    LOG.info("Initializing the 1024 buffers with [{0}] buffers.", _1024Size);
+    _1024 = setupBuffers(1024, _1024Size);
+    LOG.info("Initializing the 8192 buffers with [{0}] buffers.", _8192Size);
+    _8192 = setupBuffers(8192, _8192Size);
+    _blurShardBuffercacheLost = metrics.blurShardBuffercacheLost;
+    _blurShardBuffercacheAllocate1024 = metrics.blurShardBuffercacheAllocate1024;
+    _blurShardBuffercacheAllocate8192 = metrics.blurShardBuffercacheAllocate8192;
+    _blurShardBuffercacheAllocateOther = metrics.blurShardBuffercacheAllocateOther;
+  }
+
+  private static BlockingQueue<byte[]> setupBuffers(int bufferSize, int count) {
+    BlockingQueue<byte[]> queue = new ArrayBlockingQueue<byte[]>(count);
+    for (int i = 0; i < count; i++) {
+      queue.add(new byte[bufferSize]);
+    }
+    return queue;
+  }
+
+  public static byte[] takeBuffer(int bufferSize) {
+    switch (bufferSize) {
+    case 1024:
+      return newBuffer1024(_1024.poll());
+    case 8192:
+      return newBuffer8192(_8192.poll());
+    default:
+      return newBuffer(bufferSize);
+    }
+  }
+
+  public static void putBuffer(byte[] buffer) {
+    if (buffer == null) {
+      return;
+    }
+    int bufferSize = buffer.length;
+    switch (bufferSize) {
+    case 1024:
+      checkReturn(_1024.offer(buffer));
+      return;
+    case 8192:
+      checkReturn(_8192.offer(buffer));
+      return;
+    }
+  }
+
+  private static void checkReturn(boolean offer) {
+    if (!offer) {
+      _blurShardBuffercacheLost.incrementAndGet();
+    }
+  }
+
+  private static byte[] newBuffer1024(byte[] buf) {
+    if (buf != null) {
+      return buf;
+    }
+    _blurShardBuffercacheAllocate1024.incrementAndGet();
+    return new byte[1024];
+  }
+
+  private static byte[] newBuffer8192(byte[] buf) {
+    if (buf != null) {
+      return buf;
+    }
+    _blurShardBuffercacheAllocate8192.incrementAndGet();
+    return new byte[8192];
+  }
+
+  private static byte[] newBuffer(int size) {
+    _blurShardBuffercacheAllocateOther.incrementAndGet();
+    return new byte[size];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/39539e56/src/blur-store/src/main/java/org/apache/blur/store/buffer/ReusedBufferedIndexInput.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/buffer/ReusedBufferedIndexInput.java b/src/blur-store/src/main/java/org/apache/blur/store/buffer/ReusedBufferedIndexInput.java
new file mode 100644
index 0000000..23c09d3
--- /dev/null
+++ b/src/blur-store/src/main/java/org/apache/blur/store/buffer/ReusedBufferedIndexInput.java
@@ -0,0 +1,361 @@
+package org.apache.blur.store.buffer;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+
+/** Base implementation class for buffered {@link IndexInput}. */
+public abstract class ReusedBufferedIndexInput extends IndexInput {
+
+  // START Changed code from normal BufferedIndexInput
+  @Override
+  public final void close() throws IOException {
+    closeInternal();
+    BufferStore.putBuffer(buffer);
+    buffer = null;
+  }
+
+  protected abstract void closeInternal() throws IOException;
+
+  private void refill() throws IOException {
+    long start = bufferStart + bufferPosition;
+    long end = start + bufferSize;
+    if (end > length()) // don't read past EOF
+      end = length();
+    int newLength = (int) (end - start);
+    if (newLength <= 0)
+      throw new EOFException("read past EOF: " + this);
+
+    if (buffer == null) {
+      buffer = BufferStore.takeBuffer(bufferSize);
+      seekInternal(bufferStart);
+    }
+    readInternal(buffer, 0, newLength);
+    bufferLength = newLength;
+    bufferStart = start;
+    bufferPosition = 0;
+  }
+  
+  // The normal read buffer size defaults to 1024, but
+  // increasing this during merging seems to yield
+  // performance gains. However we don't want to increase
+  // it too much because there are quite a few
+  // BufferedIndexInputs created during merging. See
+  // LUCENE-888 for details.
+  /**
+   * A buffer size for merges set to 8192, 4k in Lucene
+   */
+  public static final int MERGE_BUFFER_SIZE = 8192;
+  
+  //END Changed code from normal BufferedIndexInput
+
+  /** Default buffer size set to 1024 */
+  public static final int BUFFER_SIZE = 1024;
+
+
+
+  private int bufferSize = BUFFER_SIZE;
+
+  protected byte[] buffer;
+
+  private long bufferStart = 0; // position in file of buffer
+  private int bufferLength = 0; // end of valid bytes
+  private int bufferPosition = 0; // next byte to read
+
+  @Override
+  public final byte readByte() throws IOException {
+    if (bufferPosition >= bufferLength)
+      refill();
+    return buffer[bufferPosition++];
+  }
+
+  public ReusedBufferedIndexInput(String resourceDesc) {
+    this(resourceDesc, BUFFER_SIZE);
+  }
+
+  public ReusedBufferedIndexInput(String resourceDesc, IOContext context) {
+    this(resourceDesc, bufferSize(context));
+  }
+
+  /** Inits BufferedIndexInput with a specific bufferSize */
+  public ReusedBufferedIndexInput(String resourceDesc, int bufferSize) {
+    super(resourceDesc);
+    checkBufferSize(bufferSize);
+    this.bufferSize = bufferSize;
+  }
+
+  /** Returns buffer size. @see #setBufferSize */
+  public final int getBufferSize() {
+    return bufferSize;
+  }
+
+  private void checkBufferSize(int bufferSize) {
+    if (bufferSize <= 0)
+      throw new IllegalArgumentException("bufferSize must be greater than 0 (got " + bufferSize + ")");
+  }
+
+  @Override
+  public final void readBytes(byte[] b, int offset, int len) throws IOException {
+    readBytes(b, offset, len, true);
+  }
+
+  @Override
+  public final void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
+
+    if (len <= (bufferLength - bufferPosition)) {
+      // the buffer contains enough data to satisfy this request
+      if (len > 0) // to allow b to be null if len is 0...
+        System.arraycopy(buffer, bufferPosition, b, offset, len);
+      bufferPosition += len;
+    } else {
+      // the buffer does not have enough data. First serve all we've got.
+      int available = bufferLength - bufferPosition;
+      if (available > 0) {
+        System.arraycopy(buffer, bufferPosition, b, offset, available);
+        offset += available;
+        len -= available;
+        bufferPosition += available;
+      }
+      // and now, read the remaining 'len' bytes:
+      if (useBuffer && len < bufferSize) {
+        // If the amount left to read is small enough, and
+        // we are allowed to use our buffer, do it in the usual
+        // buffered way: fill the buffer and copy from it:
+        refill();
+        if (bufferLength < len) {
+          // Throw an exception when refill() could not read len bytes:
+          System.arraycopy(buffer, 0, b, offset, bufferLength);
+          throw new EOFException("read past EOF: " + this);
+        } else {
+          System.arraycopy(buffer, 0, b, offset, len);
+          bufferPosition = len;
+        }
+      } else {
+        // The amount left to read is larger than the buffer
+        // or we've been asked to not use our buffer -
+        // there's no performance reason not to read it all
+        // at once. Note that unlike the previous code of
+        // this function, there is no need to do a seek
+        // here, because there's no need to reread what we
+        // had in the buffer.
+        long after = bufferStart + bufferPosition + len;
+        if (after > length())
+          throw new EOFException("read past EOF: " + this);
+        readInternal(b, offset, len);
+        bufferStart = after;
+        bufferPosition = 0;
+        bufferLength = 0; // trigger refill() on read
+      }
+    }
+  }
+
+  @Override
+  public final short readShort() throws IOException {
+    if (2 <= (bufferLength - bufferPosition)) {
+      return (short) (((buffer[bufferPosition++] & 0xFF) << 8) | (buffer[bufferPosition++] & 0xFF));
+    } else {
+      return super.readShort();
+    }
+  }
+
+  @Override
+  public final int readInt() throws IOException {
+    if (4 <= (bufferLength - bufferPosition)) {
+      return ((buffer[bufferPosition++] & 0xFF) << 24) | ((buffer[bufferPosition++] & 0xFF) << 16) | ((buffer[bufferPosition++] & 0xFF) << 8) | (buffer[bufferPosition++] & 0xFF);
+    } else {
+      return super.readInt();
+    }
+  }
+
+  @Override
+  public final long readLong() throws IOException {
+    if (8 <= (bufferLength - bufferPosition)) {
+      final int i1 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16) | ((buffer[bufferPosition++] & 0xff) << 8)
+          | (buffer[bufferPosition++] & 0xff);
+      final int i2 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16) | ((buffer[bufferPosition++] & 0xff) << 8)
+          | (buffer[bufferPosition++] & 0xff);
+      return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
+    } else {
+      return super.readLong();
+    }
+  }
+
+  @Override
+  public final int readVInt() throws IOException {
+    if (5 <= (bufferLength - bufferPosition)) {
+      byte b = buffer[bufferPosition++];
+      if (b >= 0)
+        return b;
+      int i = b & 0x7F;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7F) << 7;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7F) << 14;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7F) << 21;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
+      i |= (b & 0x0F) << 28;
+      if ((b & 0xF0) == 0)
+        return i;
+      throw new IOException("Invalid vInt detected (too many bits)");
+    } else {
+      return super.readVInt();
+    }
+  }
+
+  @Override
+  public final long readVLong() throws IOException {
+    if (9 <= bufferLength - bufferPosition) {
+      byte b = buffer[bufferPosition++];
+      if (b >= 0)
+        return b;
+      long i = b & 0x7FL;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 7;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 14;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 21;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 28;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 35;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 42;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 49;
+      if (b >= 0)
+        return i;
+      b = buffer[bufferPosition++];
+      i |= (b & 0x7FL) << 56;
+      if (b >= 0)
+        return i;
+      throw new IOException("Invalid vLong detected (negative values disallowed)");
+    } else {
+      return super.readVLong();
+    }
+  }
+
+  /**
+   * Expert: implements buffer refill. Reads bytes from the current position in
+   * the input.
+   * 
+   * @param b
+   *          the array to read bytes into
+   * @param offset
+   *          the offset in the array to start storing bytes
+   * @param length
+   *          the number of bytes to read
+   */
+  protected abstract void readInternal(byte[] b, int offset, int length) throws IOException;
+
+  @Override
+  public final long getFilePointer() {
+    return bufferStart + bufferPosition;
+  }
+
+  @Override
+  public final void seek(long pos) throws IOException {
+    if (pos >= bufferStart && pos < (bufferStart + bufferLength))
+      bufferPosition = (int) (pos - bufferStart); // seek within buffer
+    else {
+      bufferStart = pos;
+      bufferPosition = 0;
+      bufferLength = 0; // trigger refill() on read()
+      seekInternal(pos);
+    }
+  }
+
+  /**
+   * Expert: implements seek. Sets current position in this file, where the next
+   * {@link #readInternal(byte[],int,int)} will occur.
+   * 
+   * @see #readInternal(byte[],int,int)
+   */
+  protected abstract void seekInternal(long pos) throws IOException;
+
+  @Override
+  public ReusedBufferedIndexInput clone() {
+    ReusedBufferedIndexInput clone = (ReusedBufferedIndexInput) super.clone();
+
+    clone.buffer = null;
+    clone.bufferLength = 0;
+    clone.bufferPosition = 0;
+    clone.bufferStart = getFilePointer();
+
+    return clone;
+  }
+
+  /**
+   * Flushes the in-memory bufer to the given output, copying at most
+   * <code>numBytes</code>.
+   * <p>
+   * <b>NOTE:</b> this method does not refill the buffer, however it does
+   * advance the buffer position.
+   * 
+   * @return the number of bytes actually flushed from the in-memory buffer.
+   */
+  protected final int flushBuffer(IndexOutput out, long numBytes) throws IOException {
+    int toCopy = bufferLength - bufferPosition;
+    if (toCopy > numBytes) {
+      toCopy = (int) numBytes;
+    }
+    if (toCopy > 0) {
+      out.writeBytes(buffer, bufferPosition, toCopy);
+      bufferPosition += toCopy;
+    }
+    return toCopy;
+  }
+
+  /**
+   * Returns default buffer sizes for the given {@link IOContext}
+   */
+  public static int bufferSize(IOContext context) {
+    switch (context.context) {
+    case MERGE:
+      return MERGE_BUFFER_SIZE;
+    default:
+      return BUFFER_SIZE;
+    }
+  }
+
+}


Mime
View raw message