incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [26/51] [partial] Initial repackage to org.apache.
Date Mon, 03 Sep 2012 03:17:12 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurRecordWriterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurRecordWriterTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurRecordWriterTest.java
new file mode 100644
index 0000000..9cdf959
--- /dev/null
+++ b/src/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurRecordWriterTest.java
@@ -0,0 +1,87 @@
+package org.apache.blur.mapreduce.lib;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.blur.mapreduce.BlurRecord;
+import org.apache.blur.mapreduce.lib.BlurRecordWriter;
+import org.apache.blur.store.hdfs.HdfsDirectory;
+import org.apache.blur.utils.BlurConstants;
+import org.apache.blur.utils.BlurUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.lucene.index.IndexReader;
+import org.junit.Test;
+
+
+public class BlurRecordWriterTest {
+
+  @Test
+  public void testBlurRecordWriter() throws IOException, InterruptedException {
+    JobID jobId = new JobID();
+    TaskID tId = new TaskID(jobId, false, 13);
+    TaskAttemptID taskId = new TaskAttemptID(tId, 0);
+    Configuration conf = new Configuration();
+    String pathStr = "./tmp/output-record-writer-test-newapi";
+    rm(new File(pathStr));
+    conf.set("mapred.output.dir", pathStr);
+    TaskAttemptContext context = new TaskAttemptContext(conf, taskId);
+    BlurRecordWriter writer = new BlurRecordWriter(context);
+
+    Text key = new Text();
+    BlurRecord value = new BlurRecord();
+
+    for (int i = 0; i < 10; i++) {
+      String rowId = UUID.randomUUID().toString();
+      key.set(rowId);
+      value.setFamily("cf");
+      value.setRowId(rowId);
+      value.setRecordId(UUID.randomUUID().toString());
+      value.addColumn("name", "value");
+      writer.write(key, value);
+    }
+
+    writer.close(context);
+
+    // assert index exists and has document
+
+    HdfsDirectory dir = new HdfsDirectory(new Path(pathStr, BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, 13)));
+    assertTrue(IndexReader.indexExists(dir));
+    IndexReader reader = IndexReader.open(dir);
+    assertEquals(10, reader.numDocs());
+  }
+
+  private void rm(File file) {
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        rm(f);
+      }
+    }
+    file.delete();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-store/pom.xml b/src/blur-store/pom.xml
index 6653d48..2c53ec1 100644
--- a/src/blur-store/pom.xml
+++ b/src/blur-store/pom.xml
@@ -20,19 +20,19 @@ under the License.
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<parent>
-		<groupId>com.nearinfinity.blur</groupId>
+		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
 		<version>0.1.3</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
-	<groupId>com.nearinfinity.blur</groupId>
+	<groupId>org.apache.blur</groupId>
 	<artifactId>blur-store</artifactId>
 	<packaging>jar</packaging>
 	<name>Blur Store</name>
 
 	<dependencies>
 		<dependency>
-			<groupId>com.nearinfinity.blur</groupId>
+			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-util</artifactId>
 			<version>0.1.3</version>
 		</dependency>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/index/IndexWriter.java b/src/blur-store/src/main/java/com/nearinfinity/blur/index/IndexWriter.java
deleted file mode 100644
index 5cca4db..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/index/IndexWriter.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package com.nearinfinity.blur.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.lang.reflect.Field;
-
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockObtainFailedException;
-
-public class IndexWriter extends org.apache.lucene.index.IndexWriter {
-
-  private Lock internalLock;
-
-  public IndexWriter(Directory d, IndexWriterConfig conf) throws CorruptIndexException, LockObtainFailedException, IOException {
-    super(d, conf);
-    try {
-      internalLock = getInternalLock();
-    } catch (Exception e) {
-      throw new RuntimeException("Could not get the write lock instance.", e);
-    }
-  }
-
-  private Lock getInternalLock() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException {
-    Field field = org.apache.lucene.index.IndexWriter.class.getDeclaredField("writeLock");
-    field.setAccessible(true);
-    return (Lock) field.get(this);
-  }
-
-  @Override
-  protected void doAfterFlush() throws IOException {
-    super.doAfterFlush();
-    if (!internalLock.isLocked()) {
-      throw new IOException("Lock [" + internalLock + "] no longer has write lock.");
-    }
-  }
-
-  @Override
-  protected void doBeforeFlush() throws IOException {
-    super.doBeforeFlush();
-    if (!internalLock.isLocked()) {
-      throw new IOException("Lock [" + internalLock + "] no longer has write lock.");
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java b/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
deleted file mode 100644
index dfa3c9b..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
+++ /dev/null
@@ -1,334 +0,0 @@
-package com.nearinfinity.blur.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.TermFreqVector;
-import org.apache.lucene.index.TermPositions;
-import org.apache.lucene.index.TermVectorMapper;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-
-@SuppressWarnings("deprecation")
-public class EscapeRewrite {
-
-  public static void main(String[] args) throws CorruptIndexException, IOException {
-    Directory directory = FSDirectory.open(new File("/Users/amccurry/Documents/workspace/low-lat-lucene-rt/index"));
-    AtomicBoolean running = new AtomicBoolean(true);
-    IndexReader indexReader = IndexReader.open(directory);
-    // IndexReader reader = indexReader;
-    IndexReader reader = wrap(indexReader, running);
-    Query query = new WildcardQuery(new Term("id", "*0*"));
-    // Query query = new TermQuery(new Term("id","0"));
-    escapeIn(running, TimeUnit.SECONDS.toMillis(5));
-    IndexSearcher searcher = new IndexSearcher(reader);
-    long s1 = System.nanoTime();
-    Query rewrite = searcher.rewrite(query);
-    long e1 = System.nanoTime();
-
-    long s2 = System.nanoTime();
-    TopDocs topDocs = searcher.search(rewrite, 10);
-    long e2 = System.nanoTime();
-
-    System.out.println((e1 - s1) / 1000000.0 + " " + rewrite);
-    System.out.println((e2 - s2) / 1000000.0 + " " + topDocs.totalHits);
-  }
-
-  private static void escapeIn(final AtomicBoolean running, final long millis) {
-    new Thread(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          Thread.sleep(millis);
-        } catch (InterruptedException e) {
-          e.printStackTrace();
-        }
-        running.set(false);
-      }
-    }).start();
-  }
-
-  public static IndexReader wrap(IndexReader reader, AtomicBoolean running) {
-    return new IndexReaderRewriteEscape(reader, running);
-  }
-
-  public static class IndexReaderRewriteEscape extends IndexReader {
-    private IndexReader reader;
-    private AtomicBoolean running;
-
-    public IndexReaderRewriteEscape(IndexReader reader, AtomicBoolean running) {
-      this.reader = reader;
-      this.running = running;
-    }
-
-    public String toString() {
-      return reader.toString();
-    }
-
-    public IndexReader reopen() throws CorruptIndexException, IOException {
-      return reader.reopen();
-    }
-
-    public IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
-      return reader.reopen(openReadOnly);
-    }
-
-    public IndexReader reopen(IndexCommit commit) throws CorruptIndexException, IOException {
-      return reader.reopen(commit);
-    }
-
-    public IndexReader reopen(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
-      return reader.reopen(writer, applyAllDeletes);
-    }
-
-    public Directory directory() {
-      return reader.directory();
-    }
-
-    public long getVersion() {
-      return reader.getVersion();
-    }
-
-    public boolean isOptimized() {
-      return reader.isOptimized();
-    }
-
-    public TermFreqVector[] getTermFreqVectors(int docNumber) throws IOException {
-      return reader.getTermFreqVectors(docNumber);
-    }
-
-    public TermFreqVector getTermFreqVector(int docNumber, String field) throws IOException {
-      return reader.getTermFreqVector(docNumber, field);
-    }
-
-    public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException {
-      reader.getTermFreqVector(docNumber, field, mapper);
-    }
-
-    public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
-      reader.getTermFreqVector(docNumber, mapper);
-    }
-
-    public int numDocs() {
-      return reader.numDocs();
-    }
-
-    public int maxDoc() {
-      return reader.maxDoc();
-    }
-
-    public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
-      return reader.document(n, fieldSelector);
-    }
-
-    public boolean hasDeletions() {
-      return reader.hasDeletions();
-    }
-
-    public boolean hasNorms(String field) throws IOException {
-      return reader.hasNorms(field);
-    }
-
-    public int docFreq(Term t) throws IOException {
-      return reader.docFreq(t);
-    }
-
-    public boolean equals(Object arg0) {
-      return reader.equals(arg0);
-    }
-
-    public Map<String, String> getCommitUserData() {
-      return reader.getCommitUserData();
-    }
-
-    public FieldInfos getFieldInfos() {
-      return reader.getFieldInfos();
-    }
-
-    public IndexCommit getIndexCommit() throws IOException {
-      return reader.getIndexCommit();
-    }
-
-    public Object getCoreCacheKey() {
-      return reader.getCoreCacheKey();
-    }
-
-    public Object getDeletesCacheKey() {
-      return reader.getDeletesCacheKey();
-    }
-
-    public long getUniqueTermCount() throws IOException {
-      return reader.getUniqueTermCount();
-    }
-
-    public int getTermInfosIndexDivisor() {
-      return reader.getTermInfosIndexDivisor();
-    }
-
-    public int hashCode() {
-      return reader.hashCode();
-    }
-
-    public boolean isCurrent() throws CorruptIndexException, IOException {
-      return reader.isCurrent();
-    }
-
-    public boolean isDeleted(int n) {
-      return reader.isDeleted(n);
-    }
-
-    public byte[] norms(String field) throws IOException {
-      return reader.norms(field);
-    }
-
-    public void norms(String field, byte[] bytes, int offset) throws IOException {
-      reader.norms(field, bytes, offset);
-    }
-
-    public TermDocs termDocs(Term term) throws IOException {
-      return reader.termDocs(term);
-    }
-
-    public TermDocs termDocs() throws IOException {
-      return reader.termDocs();
-    }
-
-    public TermPositions termPositions() throws IOException {
-      return reader.termPositions();
-    }
-
-    public Object clone() {
-      IndexReaderRewriteEscape clone = (IndexReaderRewriteEscape) super.clone();
-      clone.reader = (IndexReader) reader.clone();
-      return clone;
-    }
-
-    public IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
-      IndexReaderRewriteEscape clone = (IndexReaderRewriteEscape) super.clone();
-      clone.reader = reader.clone(openReadOnly);
-      return clone;
-    }
-
-    public IndexReader[] getSequentialSubReaders() {
-      return wrap(reader.getSequentialSubReaders(), running);
-    }
-
-    public TermEnum terms() throws IOException {
-      return wrap(reader.terms(), running);
-    }
-
-    public TermEnum terms(Term t) throws IOException {
-      return wrap(reader.terms(t), running);
-    }
-
-    @Override
-    protected void doSetNorm(int doc, String field, byte value) throws CorruptIndexException, IOException {
-      reader.setNorm(doc, field, value);
-    }
-
-    @Override
-    protected void doDelete(int docNum) throws CorruptIndexException, IOException {
-      reader.deleteDocument(docNum);
-    }
-
-    @Override
-    protected void doUndeleteAll() throws CorruptIndexException, IOException {
-      reader.undeleteAll();
-    }
-
-    @Override
-    protected void doCommit(Map<String, String> commitUserData) throws IOException {
-      reader.commit(commitUserData);
-    }
-
-    @Override
-    protected void doClose() throws IOException {
-      reader.close();
-    }
-  }
-
-  public static TermEnum wrap(final TermEnum terms, final AtomicBoolean running) {
-    return new TermEnum() {
-
-      private int count = 0;
-      private boolean quit = false;
-
-      @Override
-      public Term term() {
-        Term term = terms.term();
-        System.out.println(term);
-        return term;
-      }
-
-      @Override
-      public boolean next() throws IOException {
-        if (quit) {
-          return false;
-        }
-        if (count >= 10000) {
-          if (!running.get()) {
-            quit = true;
-          }
-          count = 0;
-        }
-        count++;
-        return terms.next();
-      }
-
-      @Override
-      public int docFreq() {
-        return terms.docFreq();
-      }
-
-      @Override
-      public void close() throws IOException {
-        terms.close();
-      }
-    };
-  }
-
-  public static IndexReader[] wrap(IndexReader[] sequentialSubReaders, AtomicBoolean running) {
-    if (sequentialSubReaders == null) {
-      return null;
-    }
-    IndexReader[] result = new IndexReader[sequentialSubReaders.length];
-    for (int i = 0; i < sequentialSubReaders.length; i++) {
-      result[i] = wrap(sequentialSubReaders[i], running);
-    }
-    return result;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/LuceneConstant.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/LuceneConstant.java b/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/LuceneConstant.java
deleted file mode 100644
index b38d85e..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/LuceneConstant.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.nearinfinity.blur.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.lucene.util.Version;
-
-public class LuceneConstant {
-
-  public static final Version LUCENE_VERSION = Version.LUCENE_36;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/BufferStore.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/BufferStore.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/BufferStore.java
deleted file mode 100644
index a0d8206..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/BufferStore.java
+++ /dev/null
@@ -1,112 +0,0 @@
-package com.nearinfinity.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.nearinfinity.blur.BlurConfiguration;
-import com.nearinfinity.blur.log.Log;
-import com.nearinfinity.blur.log.LogFactory;
-import com.nearinfinity.blur.metrics.BlurMetrics;
-
-public class BufferStore {
-
-  private static final Log LOG = LogFactory.getLog(BufferStore.class);
-
-  private static BlockingQueue<byte[]> _1024 = setupBuffers(1024, 1);
-  private static BlockingQueue<byte[]> _8192 = setupBuffers(8192, 1);
-  public static AtomicLong _blurShardBuffercacheLost = new AtomicLong();
-  public static AtomicLong _blurShardBuffercacheAllocate1024 = new AtomicLong();
-  public static AtomicLong _blurShardBuffercacheAllocate8192 = new AtomicLong();
-  public static AtomicLong _blurShardBuffercacheAllocateOther = new AtomicLong();
-
-  public static void init(BlurConfiguration configuration, BlurMetrics metrics) {
-    int _1024Size = configuration.getInt("blur.shard.buffercache.1024", 8192);
-    int _8192Size = configuration.getInt("blur.shard.buffercache.8192", 8192);
-    LOG.info("Initializing the 1024 buffers with [{0}] buffers.", _1024Size);
-    _1024 = setupBuffers(1024, _1024Size);
-    LOG.info("Initializing the 8192 buffers with [{0}] buffers.", _8192Size);
-    _8192 = setupBuffers(8192, _8192Size);
-    _blurShardBuffercacheLost = metrics.blurShardBuffercacheLost;
-    _blurShardBuffercacheAllocate1024 = metrics.blurShardBuffercacheAllocate1024;
-    _blurShardBuffercacheAllocate8192 = metrics.blurShardBuffercacheAllocate8192;
-    _blurShardBuffercacheAllocateOther = metrics.blurShardBuffercacheAllocateOther;
-  }
-
-  private static BlockingQueue<byte[]> setupBuffers(int bufferSize, int count) {
-    BlockingQueue<byte[]> queue = new ArrayBlockingQueue<byte[]>(count);
-    for (int i = 0; i < count; i++) {
-      queue.add(new byte[bufferSize]);
-    }
-    return queue;
-  }
-
-  public static byte[] takeBuffer(int bufferSize) {
-    switch (bufferSize) {
-    case 1024:
-      return newBuffer1024(_1024.poll());
-    case 8192:
-      return newBuffer8192(_8192.poll());
-    default:
-      return newBuffer(bufferSize);
-    }
-  }
-
-  public static void putBuffer(byte[] buffer) {
-    if (buffer == null) {
-      return;
-    }
-    int bufferSize = buffer.length;
-    switch (bufferSize) {
-    case 1024:
-      checkReturn(_1024.offer(buffer));
-      return;
-    case 8192:
-      checkReturn(_8192.offer(buffer));
-      return;
-    }
-  }
-
-  private static void checkReturn(boolean offer) {
-    if (!offer) {
-      _blurShardBuffercacheLost.incrementAndGet();
-    }
-  }
-
-  private static byte[] newBuffer1024(byte[] buf) {
-    if (buf != null) {
-      return buf;
-    }
-    _blurShardBuffercacheAllocate1024.incrementAndGet();
-    return new byte[1024];
-  }
-
-  private static byte[] newBuffer8192(byte[] buf) {
-    if (buf != null) {
-      return buf;
-    }
-    _blurShardBuffercacheAllocate8192.incrementAndGet();
-    return new byte[8192];
-  }
-
-  private static byte[] newBuffer(int size) {
-    _blurShardBuffercacheAllocateOther.incrementAndGet();
-    return new byte[size];
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/CustomBufferedIndexInput.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/CustomBufferedIndexInput.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/CustomBufferedIndexInput.java
deleted file mode 100644
index 8fb7f74..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/CustomBufferedIndexInput.java
+++ /dev/null
@@ -1,277 +0,0 @@
-package com.nearinfinity.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-
-public abstract class CustomBufferedIndexInput extends IndexInput {
-
-  public static final int BUFFER_SIZE = 1024;
-
-  private int bufferSize = BUFFER_SIZE;
-
-  protected byte[] buffer;
-
-  private long bufferStart = 0; // position in file of buffer
-  private int bufferLength = 0; // end of valid bytes
-  private int bufferPosition = 0; // next byte to read
-
-  @Override
-  public byte readByte() throws IOException {
-    if (bufferPosition >= bufferLength)
-      refill();
-    return buffer[bufferPosition++];
-  }
-
-  public CustomBufferedIndexInput(String resourceDesc) {
-    this(resourceDesc, BUFFER_SIZE);
-  }
-
-  public CustomBufferedIndexInput(String resourceDesc, int bufferSize) {
-    super(resourceDesc);
-    checkBufferSize(bufferSize);
-    this.bufferSize = bufferSize;
-  }
-
-  private void checkBufferSize(int bufferSize) {
-    if (bufferSize <= 0)
-      throw new IllegalArgumentException("bufferSize must be greater than 0 (got " + bufferSize + ")");
-  }
-
-  @Override
-  public void readBytes(byte[] b, int offset, int len) throws IOException {
-    readBytes(b, offset, len, true);
-  }
-
-  @Override
-  public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
-
-    if (len <= (bufferLength - bufferPosition)) {
-      // the buffer contains enough data to satisfy this request
-      if (len > 0) // to allow b to be null if len is 0...
-        System.arraycopy(buffer, bufferPosition, b, offset, len);
-      bufferPosition += len;
-    } else {
-      // the buffer does not have enough data. First serve all we've got.
-      int available = bufferLength - bufferPosition;
-      if (available > 0) {
-        System.arraycopy(buffer, bufferPosition, b, offset, available);
-        offset += available;
-        len -= available;
-        bufferPosition += available;
-      }
-      // and now, read the remaining 'len' bytes:
-      if (useBuffer && len < bufferSize) {
-        // If the amount left to read is small enough, and
-        // we are allowed to use our buffer, do it in the usual
-        // buffered way: fill the buffer and copy from it:
-        refill();
-        if (bufferLength < len) {
-          // Throw an exception when refill() could not read len bytes:
-          System.arraycopy(buffer, 0, b, offset, bufferLength);
-          throw new IOException("read past EOF");
-        } else {
-          System.arraycopy(buffer, 0, b, offset, len);
-          bufferPosition = len;
-        }
-      } else {
-        // The amount left to read is larger than the buffer
-        // or we've been asked to not use our buffer -
-        // there's no performance reason not to read it all
-        // at once. Note that unlike the previous code of
-        // this function, there is no need to do a seek
-        // here, because there's no need to reread what we
-        // had in the buffer.
-        long after = bufferStart + bufferPosition + len;
-        if (after > length())
-          throw new IOException("read past EOF");
-        readInternal(b, offset, len);
-        bufferStart = after;
-        bufferPosition = 0;
-        bufferLength = 0; // trigger refill() on read
-      }
-    }
-  }
-
-  @Override
-  public int readInt() throws IOException {
-    if (4 <= (bufferLength - bufferPosition)) {
-      return ((buffer[bufferPosition++] & 0xFF) << 24) | ((buffer[bufferPosition++] & 0xFF) << 16) | ((buffer[bufferPosition++] & 0xFF) << 8) | (buffer[bufferPosition++] & 0xFF);
-    } else {
-      return super.readInt();
-    }
-  }
-
-  @Override
-  public long readLong() throws IOException {
-    if (8 <= (bufferLength - bufferPosition)) {
-      final int i1 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16) | ((buffer[bufferPosition++] & 0xff) << 8)
-          | (buffer[bufferPosition++] & 0xff);
-      final int i2 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16) | ((buffer[bufferPosition++] & 0xff) << 8)
-          | (buffer[bufferPosition++] & 0xff);
-      return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
-    } else {
-      return super.readLong();
-    }
-  }
-
-  @Override
-  public int readVInt() throws IOException {
-    if (5 <= (bufferLength - bufferPosition)) {
-      byte b = buffer[bufferPosition++];
-      int i = b & 0x7F;
-      for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-        b = buffer[bufferPosition++];
-        i |= (b & 0x7F) << shift;
-      }
-      return i;
-    } else {
-      return super.readVInt();
-    }
-  }
-
-  @Override
-  public long readVLong() throws IOException {
-    if (9 <= bufferLength - bufferPosition) {
-      byte b = buffer[bufferPosition++];
-      long i = b & 0x7F;
-      for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-        b = buffer[bufferPosition++];
-        i |= (b & 0x7FL) << shift;
-      }
-      return i;
-    } else {
-      return super.readVLong();
-    }
-  }
-
-  private void refill() throws IOException {
-    long start = bufferStart + bufferPosition;
-    long end = start + bufferSize;
-    if (end > length()) // don't read past EOF
-      end = length();
-    int newLength = (int) (end - start);
-    if (newLength <= 0)
-      throw new IOException("read past EOF");
-
-    if (buffer == null) {
-      buffer = BufferStore.takeBuffer(bufferSize);
-      seekInternal(bufferStart);
-    }
-    readInternal(buffer, 0, newLength);
-    bufferLength = newLength;
-    bufferStart = start;
-    bufferPosition = 0;
-  }
-
-  @Override
-  public final void close() throws IOException {
-    closeInternal();
-    BufferStore.putBuffer(buffer);
-    buffer = null;
-  }
-
-  protected abstract void closeInternal() throws IOException;
-
-  /**
-   * Expert: implements buffer refill. Reads bytes from the current position in
-   * the input.
-   * 
-   * @param b
-   *          the array to read bytes into
-   * @param offset
-   *          the offset in the array to start storing bytes
-   * @param length
-   *          the number of bytes to read
-   */
-  protected abstract void readInternal(byte[] b, int offset, int length) throws IOException;
-
-  @Override
-  public long getFilePointer() {
-    return bufferStart + bufferPosition;
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    if (pos >= bufferStart && pos < (bufferStart + bufferLength))
-      bufferPosition = (int) (pos - bufferStart); // seek within buffer
-    else {
-      bufferStart = pos;
-      bufferPosition = 0;
-      bufferLength = 0; // trigger refill() on read()
-      seekInternal(pos);
-    }
-  }
-
-  /**
-   * Expert: implements seek. Sets current position in this file, where the next
-   * {@link #readInternal(byte[],int,int)} will occur.
-   * 
-   * @see #readInternal(byte[],int,int)
-   */
-  protected abstract void seekInternal(long pos) throws IOException;
-
-  @Override
-  public Object clone() {
-    CustomBufferedIndexInput clone = (CustomBufferedIndexInput) super.clone();
-
-    clone.buffer = null;
-    clone.bufferLength = 0;
-    clone.bufferPosition = 0;
-    clone.bufferStart = getFilePointer();
-
-    return clone;
-  }
-
-  /**
-   * Flushes the in-memory bufer to the given output, copying at most
-   * <code>numBytes</code>.
-   * <p>
-   * <b>NOTE:</b> this method does not refill the buffer, however it does
-   * advance the buffer position.
-   * 
-   * @return the number of bytes actually flushed from the in-memory buffer.
-   */
-  protected int flushBuffer(IndexOutput out, long numBytes) throws IOException {
-    int toCopy = bufferLength - bufferPosition;
-    if (toCopy > numBytes) {
-      toCopy = (int) numBytes;
-    }
-    if (toCopy > 0) {
-      out.writeBytes(buffer, bufferPosition, toCopy);
-      bufferPosition += toCopy;
-    }
-    return toCopy;
-  }
-
-  @Override
-  public void copyBytes(IndexOutput out, long numBytes) throws IOException {
-    assert numBytes >= 0 : "numBytes=" + numBytes;
-
-    while (numBytes > 0) {
-      if (bufferLength == bufferPosition) {
-        refill();
-      }
-      numBytes -= flushBuffer(out, numBytes);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
deleted file mode 100644
index 711c874..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCache.java
+++ /dev/null
@@ -1,189 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.nio.ByteBuffer;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.EvictionListener;
-import com.nearinfinity.blur.metrics.BlurMetrics;
-
-public class BlockCache {
-
-  public static final int _128M = 134217728;
-  public static final int _32K = 32768;
-  private final ConcurrentMap<BlockCacheKey, BlockCacheLocation> _cache;
-  private final ByteBuffer[] _slabs;
-  private final BlockLocks[] _locks;
-  private final AtomicInteger[] _lockCounters;
-  private final int _blockSize;
-  private final int _numberOfBlocksPerSlab;
-  private final int _maxEntries;
-  private final BlurMetrics _metrics;
-
-  public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory) {
-    this(metrics, directAllocation, totalMemory, _128M);
-  }
-
-  public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory, int slabSize) {
-    this(metrics, directAllocation, totalMemory, slabSize, _32K);
-  }
-
-  public BlockCache(BlurMetrics metrics, boolean directAllocation, long totalMemory, int slabSize, int blockSize) {
-    _metrics = metrics;
-    _numberOfBlocksPerSlab = slabSize / blockSize;
-    int numberOfSlabs = (int) (totalMemory / slabSize);
-
-    _slabs = new ByteBuffer[numberOfSlabs];
-    _locks = new BlockLocks[numberOfSlabs];
-    _lockCounters = new AtomicInteger[numberOfSlabs];
-    _maxEntries = (_numberOfBlocksPerSlab * numberOfSlabs) - 1;
-    for (int i = 0; i < numberOfSlabs; i++) {
-      if (directAllocation) {
-        _slabs[i] = ByteBuffer.allocateDirect(_numberOfBlocksPerSlab * blockSize);
-      } else {
-        _slabs[i] = ByteBuffer.allocate(_numberOfBlocksPerSlab * blockSize);
-      }
-      _locks[i] = new BlockLocks(_numberOfBlocksPerSlab);
-      _lockCounters[i] = new AtomicInteger();
-    }
-
-    EvictionListener<BlockCacheKey, BlockCacheLocation> listener = new EvictionListener<BlockCacheKey, BlockCacheLocation>() {
-      @Override
-      public void onEviction(BlockCacheKey key, BlockCacheLocation location) {
-        releaseLocation(location);
-      }
-    };
-    _cache = new ConcurrentLinkedHashMap.Builder<BlockCacheKey, BlockCacheLocation>().maximumWeightedCapacity(_maxEntries).listener(listener).build();
-    _blockSize = blockSize;
-  }
-
-  private void releaseLocation(BlockCacheLocation location) {
-    if (location == null) {
-      return;
-    }
-    int slabId = location.getSlabId();
-    int block = location.getBlock();
-    location.setRemoved(true);
-    _locks[slabId].clear(block);
-    _lockCounters[slabId].decrementAndGet();
-    _metrics.blockCacheEviction.incrementAndGet();
-    _metrics.blockCacheSize.decrementAndGet();
-  }
-
-  public boolean store(BlockCacheKey blockCacheKey, byte[] data) {
-    checkLength(data);
-    BlockCacheLocation location = _cache.get(blockCacheKey);
-    boolean newLocation = false;
-    if (location == null) {
-      newLocation = true;
-      location = new BlockCacheLocation();
-      if (!findEmptyLocation(location)) {
-        return false;
-      }
-    }
-    if (location.isRemoved()) {
-      return false;
-    }
-    int slabId = location.getSlabId();
-    int offset = location.getBlock() * _blockSize;
-    ByteBuffer slab = getSlab(slabId);
-    slab.position(offset);
-    slab.put(data, 0, _blockSize);
-    if (newLocation) {
-      releaseLocation(_cache.put(blockCacheKey.clone(), location));
-      _metrics.blockCacheSize.incrementAndGet();
-    }
-    return true;
-  }
-
-  public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer, int blockOffset, int off, int length) {
-    BlockCacheLocation location = _cache.get(blockCacheKey);
-    if (location == null) {
-      return false;
-    }
-    if (location.isRemoved()) {
-      return false;
-    }
-    int slabId = location.getSlabId();
-    int offset = location.getBlock() * _blockSize;
-    location.touch();
-    ByteBuffer slab = getSlab(slabId);
-    slab.position(offset + blockOffset);
-    slab.get(buffer, off, length);
-    return true;
-  }
-
-  public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer) {
-    checkLength(buffer);
-    return fetch(blockCacheKey, buffer, 0, 0, _blockSize);
-  }
-
-  private boolean findEmptyLocation(BlockCacheLocation location) {
-    // This is a tight loop that will try and find a location to
-    // place the block before giving up
-    for (int j = 0; j < 10; j++) {
-      OUTER: for (int slabId = 0; slabId < _slabs.length; slabId++) {
-        AtomicInteger bitSetCounter = _lockCounters[slabId];
-        BlockLocks bitSet = _locks[slabId];
-        if (bitSetCounter.get() == _numberOfBlocksPerSlab) {
-          // if bitset is full
-          continue OUTER;
-        }
-        // this check needs to spin, if a lock was attempted but not obtained
-        // the rest of the slab should not be skipped
-        int bit = bitSet.nextClearBit(0);
-        INNER: while (bit != -1) {
-          if (bit >= _numberOfBlocksPerSlab) {
-            // bit set is full
-            continue OUTER;
-          }
-          if (!bitSet.set(bit)) {
-            // lock was not obtained
-            // this restarts at 0 because another block could have been unlocked
-            // while this was executing
-            bit = bitSet.nextClearBit(0);
-            continue INNER;
-          } else {
-            // lock obtained
-            location.setSlabId(slabId);
-            location.setBlock(bit);
-            bitSetCounter.incrementAndGet();
-            return true;
-          }
-        }
-      }
-    }
-    return false;
-  }
-
-  private void checkLength(byte[] buffer) {
-    if (buffer.length != _blockSize) {
-      throw new RuntimeException("Buffer wrong size, expecting [" + _blockSize + "] got [" + buffer.length + "]");
-    }
-  }
-
-  private ByteBuffer getSlab(int slabId) {
-    return _slabs[slabId].duplicate();
-  }
-
-  public int getSize() {
-    return _cache.size();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheKey.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheKey.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheKey.java
deleted file mode 100644
index 64dd703..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheKey.java
+++ /dev/null
@@ -1,73 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-public class BlockCacheKey implements Cloneable {
-
-  private long _block;
-  private int _file;
-
-  public long getBlock() {
-    return _block;
-  }
-
-  public int getFile() {
-    return _file;
-  }
-
-  public void setBlock(long block) {
-    _block = block;
-  }
-
-  public void setFile(int file) {
-    _file = file;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + (int) (_block ^ (_block >>> 32));
-    result = prime * result + _file;
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    BlockCacheKey other = (BlockCacheKey) obj;
-    if (_block != other._block)
-      return false;
-    if (_file != other._file)
-      return false;
-    return true;
-  }
-
-  @Override
-  public BlockCacheKey clone() {
-    try {
-      return (BlockCacheKey) super.clone();
-    } catch (CloneNotSupportedException e) {
-      throw new RuntimeException(e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
deleted file mode 100644
index 2f5c3fd..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockCacheLocation.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class BlockCacheLocation {
-
-  private int _block;
-  private int _slabId;
-  private long _lastAccess = System.currentTimeMillis();
-  private long _accesses;
-  private AtomicBoolean _removed = new AtomicBoolean(false);
-
-  public void setBlock(int block) {
-    _block = block;
-  }
-
-  public void setSlabId(int slabId) {
-    _slabId = slabId;
-  }
-
-  public int getBlock() {
-    return _block;
-  }
-
-  public int getSlabId() {
-    return _slabId;
-  }
-
-  public void touch() {
-    _lastAccess = System.currentTimeMillis();
-    _accesses++;
-  }
-
-  public long getLastAccess() {
-    return _lastAccess;
-  }
-
-  public long getNumberOfAccesses() {
-    return _accesses;
-  }
-
-  public boolean isRemoved() {
-    return _removed.get();
-  }
-
-  public void setRemoved(boolean removed) {
-    _removed.set(removed);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
deleted file mode 100644
index 314db09..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
+++ /dev/null
@@ -1,282 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Set;
-
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-
-import com.nearinfinity.blur.store.BufferStore;
-import com.nearinfinity.blur.store.CustomBufferedIndexInput;
-
-public class BlockDirectory extends Directory {
-
-  public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
-  public static final long BLOCK_MOD = 0x1FFF;
-  public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT;
-
-  public static long getBlock(long pos) {
-    return pos >>> BLOCK_SHIFT;
-  }
-
-  public static long getPosition(long pos) {
-    return pos & BLOCK_MOD;
-  }
-
-  public static long getRealPosition(long block, long positionInBlock) {
-    return (block << BLOCK_SHIFT) + positionInBlock;
-  }
-
-  public static Cache NO_CACHE = new Cache() {
-
-    @Override
-    public void update(String name, long blockId, byte[] buffer) {
-
-    }
-
-    @Override
-    public boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
-      return false;
-    }
-
-    @Override
-    public void delete(String name) {
-
-    }
-
-    @Override
-    public long size() {
-      return 0;
-    }
-  };
-
-  private Directory _directory;
-  private int _blockSize;
-  private String _dirName;
-  private Cache _cache;
-  private Set<String> _blockCacheFileTypes;
-
-  public BlockDirectory(String dirName, Directory directory) throws IOException {
-    this(dirName, directory, NO_CACHE);
-  }
-
-  public BlockDirectory(String dirName, Directory directory, Cache cache) throws IOException {
-    this(dirName, directory, cache, null);
-  }
-
-  public BlockDirectory(String dirName, Directory directory, Cache cache, Set<String> blockCacheFileTypes) throws IOException {
-    _dirName = dirName;
-    _directory = directory;
-    _blockSize = BLOCK_SIZE;
-    _cache = cache;
-    if (blockCacheFileTypes == null || blockCacheFileTypes.isEmpty()) {
-      _blockCacheFileTypes = null;
-    } else {
-      _blockCacheFileTypes = blockCacheFileTypes;
-    }
-    setLockFactory(directory.getLockFactory());
-  }
-
-  public IndexInput openInput(String name, int bufferSize) throws IOException {
-    final IndexInput source = _directory.openInput(name, _blockSize);
-    if (_blockCacheFileTypes == null || isCachableFile(name)) {
-      return new CachedIndexInput(source, _blockSize, name, getFileCacheName(name), _cache, bufferSize);
-    }
-    return source;
-  }
-
-  private boolean isCachableFile(String name) {
-    for (String ext : _blockCacheFileTypes) {
-      if (name.endsWith(ext)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  @Override
-  public IndexInput openInput(final String name) throws IOException {
-    return openInput(name, _blockSize);
-  }
-
-  static class CachedIndexInput extends CustomBufferedIndexInput {
-
-    private IndexInput _source;
-    private int _blockSize;
-    private long _fileLength;
-    private String _cacheName;
-    private Cache _cache;
-
-    public CachedIndexInput(IndexInput source, int blockSize, String name, String cacheName, Cache cache, int bufferSize) {
-      super(name, bufferSize);
-      _source = source;
-      _blockSize = blockSize;
-      _fileLength = source.length();
-      _cacheName = cacheName;
-      _cache = cache;
-    }
-
-    @Override
-    public Object clone() {
-      CachedIndexInput clone = (CachedIndexInput) super.clone();
-      clone._source = (IndexInput) _source.clone();
-      return clone;
-    }
-
-    @Override
-    public long length() {
-      return _source.length();
-    }
-
-    @Override
-    protected void seekInternal(long pos) throws IOException {
-    }
-
-    @Override
-    protected void readInternal(byte[] b, int off, int len) throws IOException {
-      long position = getFilePointer();
-      while (len > 0) {
-        int length = fetchBlock(position, b, off, len);
-        position += length;
-        len -= length;
-        off += length;
-      }
-    }
-
-    private int fetchBlock(long position, byte[] b, int off, int len) throws IOException {
-      // read whole block into cache and then provide needed data
-      long blockId = getBlock(position);
-      int blockOffset = (int) getPosition(position);
-      int lengthToReadInBlock = Math.min(len, _blockSize - blockOffset);
-      if (checkCache(blockId, blockOffset, b, off, lengthToReadInBlock)) {
-        return lengthToReadInBlock;
-      } else {
-        readIntoCacheAndResult(blockId, blockOffset, b, off, lengthToReadInBlock);
-      }
-      return lengthToReadInBlock;
-    }
-
-    private void readIntoCacheAndResult(long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) throws IOException {
-      long position = getRealPosition(blockId, 0);
-      int length = (int) Math.min(_blockSize, _fileLength - position);
-      _source.seek(position);
-
-      byte[] buf = BufferStore.takeBuffer(_blockSize);
-      _source.readBytes(buf, 0, length);
-      System.arraycopy(buf, blockOffset, b, off, lengthToReadInBlock);
-      _cache.update(_cacheName, blockId, buf);
-      BufferStore.putBuffer(buf);
-    }
-
-    private boolean checkCache(long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
-      return _cache.fetch(_cacheName, blockId, blockOffset, b, off, lengthToReadInBlock);
-    }
-
-    @Override
-    protected void closeInternal() throws IOException {
-      _source.close();
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    String[] files = listAll();
-    for (String file : files) {
-      _cache.delete(getFileCacheName(file));
-    }
-    _directory.close();
-  }
-
-  private String getFileCacheName(String name) throws IOException {
-    return _dirName + "/" + name + ":" + fileModified(name);
-  }
-
-  public void clearLock(String name) throws IOException {
-    _directory.clearLock(name);
-  }
-
-  public void copy(Directory to, String src, String dest) throws IOException {
-    _directory.copy(to, src, dest);
-  }
-
-  public LockFactory getLockFactory() {
-    return _directory.getLockFactory();
-  }
-
-  public String getLockID() {
-    return _directory.getLockID();
-  }
-
-  public Lock makeLock(String name) {
-    return _directory.makeLock(name);
-  }
-
-  public void setLockFactory(LockFactory lockFactory) throws IOException {
-    _directory.setLockFactory(lockFactory);
-  }
-
-  public void sync(Collection<String> names) throws IOException {
-    _directory.sync(names);
-  }
-
-  @SuppressWarnings("deprecation")
-  public void sync(String name) throws IOException {
-    _directory.sync(name);
-  }
-
-  public String toString() {
-    return _directory.toString();
-  }
-
-  public IndexOutput createOutput(String name) throws IOException {
-    return _directory.createOutput(name);
-  }
-
-  public void deleteFile(String name) throws IOException {
-    _cache.delete(getFileCacheName(name));
-    _directory.deleteFile(name);
-  }
-
-  public boolean fileExists(String name) throws IOException {
-    return _directory.fileExists(name);
-  }
-
-  public long fileLength(String name) throws IOException {
-    return _directory.fileLength(name);
-  }
-
-  @SuppressWarnings("deprecation")
-  public long fileModified(String name) throws IOException {
-    return _directory.fileModified(name);
-  }
-
-  public String[] listAll() throws IOException {
-    return _directory.listAll();
-  }
-
-  @SuppressWarnings("deprecation")
-  public void touchFile(String name) throws IOException {
-    _directory.touchFile(name);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryCache.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryCache.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryCache.java
deleted file mode 100644
index 5f0e297..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryCache.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.nearinfinity.blur.metrics.BlurMetrics;
-
-public class BlockDirectoryCache implements Cache {
-  private BlockCache _blockCache;
-  private AtomicInteger _counter = new AtomicInteger();
-  private Map<String, Integer> _names = new ConcurrentHashMap<String, Integer>();
-  private BlurMetrics _blurMetrics;
-
-  public BlockDirectoryCache(BlockCache blockCache, BlurMetrics blurMetrics) {
-    _blockCache = blockCache;
-    _blurMetrics = blurMetrics;
-  }
-
-  @Override
-  public void delete(String name) {
-    _names.remove(name);
-  }
-
-  @Override
-  public void update(String name, long blockId, byte[] buffer) {
-    Integer file = _names.get(name);
-    if (file == null) {
-      file = _counter.incrementAndGet();
-      _names.put(name, file);
-    }
-    BlockCacheKey blockCacheKey = new BlockCacheKey();
-    blockCacheKey.setBlock(blockId);
-    blockCacheKey.setFile(file);
-    _blockCache.store(blockCacheKey, buffer);
-  }
-
-  @Override
-  public boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
-    Integer file = _names.get(name);
-    if (file == null) {
-      return false;
-    }
-    BlockCacheKey blockCacheKey = new BlockCacheKey();
-    blockCacheKey.setBlock(blockId);
-    blockCacheKey.setFile(file);
-    boolean fetch = _blockCache.fetch(blockCacheKey, b, blockOffset, off, lengthToReadInBlock);
-    if (fetch) {
-      _blurMetrics.blockCacheHit.incrementAndGet();
-    } else {
-      _blurMetrics.blockCacheMiss.incrementAndGet();
-    }
-    return fetch;
-  }
-
-  @Override
-  public long size() {
-    return _blockCache.getSize();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockLocks.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockLocks.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockLocks.java
deleted file mode 100644
index d02685f..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockLocks.java
+++ /dev/null
@@ -1,96 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.concurrent.atomic.AtomicLongArray;
-
-import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.OpenBitSet;
-
-public class BlockLocks {
-
-  private AtomicLongArray bits;
-  private int wlen;
-
-  public BlockLocks(long numBits) {
-    int length = OpenBitSet.bits2words(numBits);
-    bits = new AtomicLongArray(length);
-    wlen = length;
-  }
-
-  /**
-   * Find the next clear bit in the bit set.
-   * 
-   * @param index
-   * @return
-   */
-  public int nextClearBit(int index) {
-    int i = index >> 6;
-    if (i >= wlen)
-      return -1;
-    int subIndex = index & 0x3f; // index within the word
-    long word = ~bits.get(i) >> subIndex; // skip all the bits to the right of
-                                          // index
-    if (word != 0) {
-      return (i << 6) + subIndex + BitUtil.ntz(word);
-    }
-    while (++i < wlen) {
-      word = ~bits.get(i);
-      if (word != 0) {
-        return (i << 6) + BitUtil.ntz(word);
-      }
-    }
-    return -1;
-  }
-
-  /**
-   * Thread safe set operation that will set the bit if and only if the bit was
-   * not previously set.
-   * 
-   * @param index
-   *          the index position to set.
-   * @return returns true if the bit was set and false if it was already set.
-   */
-  public boolean set(int index) {
-    int wordNum = index >> 6; // div 64
-    int bit = index & 0x3f; // mod 64
-    long bitmask = 1L << bit;
-    long word, oword;
-    do {
-      word = bits.get(wordNum);
-      // if set another thread stole the lock
-      if ((word & bitmask) != 0) {
-        return false;
-      }
-      oword = word;
-      word |= bitmask;
-    } while (!bits.compareAndSet(wordNum, oword, word));
-    return true;
-  }
-
-  public void clear(int index) {
-    int wordNum = index >> 6;
-    int bit = index & 0x03f;
-    long bitmask = 1L << bit;
-    long word, oword;
-    do {
-      word = bits.get(wordNum);
-      oword = word;
-      word &= ~bitmask;
-    } while (!bits.compareAndSet(wordNum, oword, word));
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/Cache.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/Cache.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/Cache.java
deleted file mode 100644
index cab4b3f..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/Cache.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package com.nearinfinity.blur.store.blockcache;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-public interface Cache {
-
-  void delete(String name);
-
-  void update(String name, long blockId, byte[] buffer);
-
-  boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock);
-
-  long size();
-
-}


Mime
View raw message