incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [31/47] git commit: Lucene 3.6.1 code fixes.
Date Mon, 03 Sep 2012 00:32:20 GMT
Lucene 3.6.1 code fixes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b8476524
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b8476524
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b8476524

Branch: refs/heads/master
Commit: b8476524b7ad5e22ebc45d96e01b9551ad939a30
Parents: dc9c1ac
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Aug 21 22:03:12 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Aug 21 22:03:12 2012 -0400

----------------------------------------------------------------------
 .../nearinfinity/blur/manager/IndexManager.java    |   94 +++++++++++++++
 .../clusterstatus/ZookeeperClusterStatus.java      |   36 +++---
 .../indexserver/DistributedIndexServer.java        |    6 +-
 .../blur/manager/indexserver/LocalIndexServer.java |    3 +-
 .../blur/manager/writer/AbstractBlurIndex.java     |   10 +-
 .../blur/manager/writer/BlurIndexReader.java       |    6 +-
 .../blur/manager/writer/BlurNRTIndex.java          |   58 +++++----
 .../manager/writer/DirectoryReferenceCounter.java  |    1 +
 .../blur/thrift/BlurControllerServer.java          |    2 +-
 .../nearinfinity/blur/thrift/BlurShardServer.java  |    9 +-
 .../blur/manager/IndexManagerTest.java             |   10 +-
 .../blur/manager/writer/BlurNRTIndexTest.java      |   19 ++--
 .../blur/search/RandomSuperQueryTest.java          |    3 +-
 .../nearinfinity/blur/search/SuperQueryTest.java   |   11 +-
 .../nearinfinity/blur/thrift/BlurClusterTest.java  |    9 +-
 .../nearinfinity/blur/mapreduce/BlurReducer.java   |   12 +-
 .../blur/mapreduce/BufferedDirectory.java          |    1 +
 .../blur/mapreduce/ProgressableDirectory.java      |    1 +
 .../nearinfinity/blur/index/DirectIODirectory.java |   79 ------------
 .../nearinfinity/blur/lucene/EscapeRewrite.java    |    4 +-
 .../blur/store/blockcache/BlockDirectory.java      |   22 +---
 .../compressed/CompressedFieldDataDirectory.java   |   22 +---
 .../blur/store/hdfs/HdfsDirectory.java             |   68 +----------
 .../blur/store/hdfs/HdfsFileReader.java            |    9 +-
 .../blur/store/BenchmarkDirectory.java             |   16 +--
 .../blur/store/BenchmarkDirectoryNrt.java          |   23 ++--
 .../blur/store/blockcache/BlockDirectoryTest.java  |    5 +-
 .../CompressedFieldDataDirectoryTest.java          |   13 +-
 28 files changed, 245 insertions(+), 307 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/IndexManager.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/IndexManager.java
index 9a25962..c4afef0 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/IndexManager.java
@@ -28,6 +28,7 @@ import static com.nearinfinity.blur.utils.RowDocumentUtil.getRow;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -642,6 +643,99 @@ public class IndexManager {
     }
   }
 
+  public void mutate(final List<RowMutation> mutations) throws BlurException, IOException {
+    Future<Void> future = _mutateExecutor.submit(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        doMutates(mutations);
+        return null;
+      }
+    });
+    try {
+      future.get();
+    } catch (InterruptedException e) {
+      throw new BException("Unknown error during mutation", e);
+    } catch (ExecutionException e) {
+      throw new BException("Unknown error during mutation", e.getCause());
+    }
+  }
+
+  private void doMutates(List<RowMutation> mutations) throws BlurException, IOException {
+    Map<String, List<RowMutation>> map = getMutatesPerTable(mutations);
+    for (Entry<String, List<RowMutation>> entry : map.entrySet()) {
+      doMutates(entry.getKey(), entry.getValue());
+    }
+  }
+
+  private void doMutates(String table, List<RowMutation> mutations) throws IOException, BlurException {
+    Map<String, BlurIndex> indexes = _indexServer.getIndexes(table);
+
+    Map<String, List<RowMutation>> mutationsByShard = new HashMap<String, List<RowMutation>>();
+
+    for (int i = 0; i < mutations.size(); i++) {
+      RowMutation mutation = mutations.get(i);
+      String shard = MutationHelper.getShardName(table, mutation.rowId, getNumberOfShards(table), _blurPartitioner);
+      List<RowMutation> list = mutationsByShard.get(shard);
+      if (list == null) {
+        list = new ArrayList<RowMutation>();
+        mutationsByShard.put(shard, list);
+      }
+      list.add(mutation);
+    }
+
+    for (Entry<String, List<RowMutation>> entry : mutationsByShard.entrySet()) {
+      executeMutates(table, entry.getKey(), indexes, entry.getValue());
+    }
+  }
+
+  private void executeMutates(String table, String shard, Map<String, BlurIndex> indexes, List<RowMutation> mutations) throws BlurException, IOException {
+    boolean waitToBeVisible = false;
+    for (int i = 0; i < mutations.size(); i++) {
+      RowMutation mutation = mutations.get(i);
+      if (mutation.waitToBeVisible) {
+        waitToBeVisible = true;
+      }
+      BlurIndex blurIndex = indexes.get(shard);
+      if (blurIndex == null) {
+        throw new BlurException("Shard [" + shard + "] in table [" + table + "] is not being served by this server.", null);
+      }
+
+      boolean waitVisiblity = false;
+      if (i + 1 == mutations.size()) {
+        waitVisiblity = waitToBeVisible;
+      }
+      RowMutationType type = mutation.rowMutationType;
+      switch (type) {
+      case REPLACE_ROW:
+        Row row = MutationHelper.getRowFromMutations(mutation.rowId, mutation.recordMutations);
+        blurIndex.replaceRow(waitVisiblity, mutation.wal, row);
+        break;
+      case UPDATE_ROW:
+        doUpdateRowMutation(mutation, blurIndex);
+        break;
+      case DELETE_ROW:
+        blurIndex.deleteRow(waitVisiblity, mutation.wal, mutation.rowId);
+        break;
+      default:
+        throw new RuntimeException("Not supported [" + type + "]");
+      }
+    }
+  }
+
+  private Map<String, List<RowMutation>> getMutatesPerTable(List<RowMutation> mutations) {
+    Map<String, List<RowMutation>> map = new HashMap<String, List<RowMutation>>();
+    for (RowMutation mutation : mutations) {
+      String table = mutation.table;
+      List<RowMutation> list = map.get(table);
+      if (list == null) {
+        list = new ArrayList<RowMutation>();
+        map.put(table, list);
+      }
+      list.add(mutation);
+    }
+    return map;
+  }
+
   private void doMutate(RowMutation mutation) throws BlurException, IOException {
     String table = mutation.table;
     Map<String, BlurIndex> indexes = _indexServer.getIndexes(table);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/clusterstatus/ZookeeperClusterStatus.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/clusterstatus/ZookeeperClusterStatus.java
index 3d52e3a..a1215c6 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/clusterstatus/ZookeeperClusterStatus.java
@@ -243,7 +243,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getClusterList [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getClusterList [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -266,7 +266,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getControllerServerList [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getControllerServerList [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -291,7 +291,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getOnlineShardServers took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getOnlineShardServers took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -320,7 +320,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getShardServerList took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getShardServerList took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -347,7 +347,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace exists took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace exists took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -372,7 +372,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace isEnabled took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace isEnabled took [" + (e - s) / 1000000.0 + " ms]");
     }
     return true;
   }
@@ -412,7 +412,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getTableDescriptor took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getTableDescriptor took [" + (e - s) / 1000000.0 + " ms]");
     }
     tableDescriptor.cluster = cluster;
     _tableDescriptorCache.put(table, tableDescriptor);
@@ -473,7 +473,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getTableList took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getTableList took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -530,7 +530,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
         throw new RuntimeException(e);
       } finally {
         long e = System.nanoTime();
-        LOG.info("trace getCluster took [" + (e - s) / 1000000.0 + " ms]");
+        LOG.debug("trace getCluster took [" + (e - s) / 1000000.0 + " ms]");
       }
     }
     return null;
@@ -568,7 +568,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace isInSafeMode took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace isInSafeMode took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -589,7 +589,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getShardCount took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getShardCount took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -616,7 +616,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace getBlockCacheFileTypes took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace getBlockCacheFileTypes took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -634,7 +634,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace isBlockCacheEnabled took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace isBlockCacheEnabled took [" + (e - s) / 1000000.0 + " ms]");
     }
     return true;
   }
@@ -661,7 +661,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace isReadOnly took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace isReadOnly took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -721,7 +721,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace createTable took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace createTable took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -746,7 +746,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace disableTable took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace disableTable took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -771,7 +771,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace enableTable took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace enableTable took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 
@@ -801,7 +801,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       throw new RuntimeException(e);
     } finally {
       long e = System.nanoTime();
-      LOG.info("trace removeTable took [" + (e - s) / 1000000.0 + " ms]");
+      LOG.debug("trace removeTable took [" + (e - s) / 1000000.0 + " ms]");
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/DistributedIndexServer.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/DistributedIndexServer.java
index 30a395c..d9b6cf9 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/DistributedIndexServer.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/DistributedIndexServer.java
@@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.TermPositions;
 import org.apache.lucene.search.Similarity;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.ReaderUtil;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -46,7 +47,6 @@ import org.apache.zookeeper.data.Stat;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
 import com.nearinfinity.blur.concurrent.Executors;
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.log.Log;
 import com.nearinfinity.blur.log.LogFactory;
 import com.nearinfinity.blur.lucene.search.FairSimilarity;
@@ -462,7 +462,7 @@ public class DistributedIndexServer extends AbstractIndexServer {
 
     BlurLockFactory lockFactory = new BlurLockFactory(_configuration, hdfsDirPath, _nodeName, BlurConstants.getPid());
 
-    DirectIODirectory directory = new HdfsDirectory(hdfsDirPath);
+    Directory directory = new HdfsDirectory(hdfsDirPath);
     directory.setLockFactory(lockFactory);
 
     TableDescriptor descriptor = _clusterStatus.getTableDescriptor(true, cluster, table);
@@ -478,7 +478,7 @@ public class DistributedIndexServer extends AbstractIndexServer {
       }
     }
 
-    DirectIODirectory dir;
+    Directory dir;
     boolean blockCacheEnabled = _clusterStatus.isBlockCacheEnabled(cluster, table);
     if (blockCacheEnabled) {
       Set<String> blockCacheFileTypes = _clusterStatus.getBlockCacheFileTypes(cluster, table);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/LocalIndexServer.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/LocalIndexServer.java
index bcdb196..7144f68 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/LocalIndexServer.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/indexserver/LocalIndexServer.java
@@ -45,7 +45,6 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MMapDirectory;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.log.Log;
 import com.nearinfinity.blur.log.LogFactory;
 import com.nearinfinity.blur.lucene.search.FairSimilarity;
@@ -146,7 +145,7 @@ public class LocalIndexServer extends AbstractIndexServer {
   private BlurIndex openIndex(String table, String shard, Directory dir) throws CorruptIndexException, IOException {
     BlurNRTIndex index = new BlurNRTIndex();
     index.setAnalyzer(getAnalyzer(table));
-    index.setDirectory(DirectIODirectory.wrap(dir));
+    index.setDirectory(dir);
     index.setShard(shard);
     index.setSimilarity(getSimilarity(table));
     index.setTable(table);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/AbstractBlurIndex.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/AbstractBlurIndex.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/AbstractBlurIndex.java
index 8582779..62e55c4 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/AbstractBlurIndex.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/AbstractBlurIndex.java
@@ -14,15 +14,15 @@ import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.Similarity;
+import org.apache.lucene.store.Directory;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
-import com.nearinfinity.blur.index.DirectIODirectory;
 
 public abstract class AbstractBlurIndex extends BlurIndex {
 
   private BlurAnalyzer _analyzer;
   private BlurIndexCloser _closer;
-  private DirectIODirectory _directory;
+  private Directory _directory;
   private IndexDeletionPolicy _indexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy();
   private AtomicReference<IndexReader> _indexReaderRef = new AtomicReference<IndexReader>();
   private AtomicBoolean _isClosed = new AtomicBoolean(false);
@@ -57,7 +57,7 @@ public abstract class AbstractBlurIndex extends BlurIndex {
     if (oldReader.isCurrent()) {
       return;
     }
-    IndexReader reader = IndexReader.openIfChanged(oldReader, true);
+    IndexReader reader = IndexReader.openIfChanged(oldReader);
     if (reader != null && oldReader != reader) {
       _indexReaderRef.set(reader);
       _closer.close(oldReader);
@@ -103,7 +103,7 @@ public abstract class AbstractBlurIndex extends BlurIndex {
     _closer = closer;
   }
 
-  public void setDirectory(DirectIODirectory directory) {
+  public void setDirectory(Directory directory) {
     _directory = directory;
   }
 
@@ -131,7 +131,7 @@ public abstract class AbstractBlurIndex extends BlurIndex {
     return _analyzer;
   }
 
-  protected DirectIODirectory getDirectory() {
+  protected Directory getDirectory() {
     return _directory;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurIndexReader.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurIndexReader.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurIndexReader.java
index 691998b..872327f 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurIndexReader.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurIndexReader.java
@@ -5,9 +5,9 @@ import java.io.IOException;
 import org.apache.lucene.analysis.KeywordAnalyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Version;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.index.IndexWriter;
 import com.nearinfinity.blur.log.Log;
 import com.nearinfinity.blur.log.LogFactory;
@@ -19,12 +19,12 @@ public class BlurIndexReader extends AbstractBlurIndex {
 
   public void init() throws IOException {
     initIndexWriterConfig();
-    DirectIODirectory directory = getDirectory();
+    Directory directory = getDirectory();
     if (!IndexReader.indexExists(directory)) {
       IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer());
       new IndexWriter(directory, conf).close();
     }
-    initIndexReader(IndexReader.open(directory, true));
+    initIndexReader(IndexReader.open(directory));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurNRTIndex.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurNRTIndex.java
index a7c62cd..3e5d551 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurNRTIndex.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/BlurNRTIndex.java
@@ -46,7 +46,7 @@ public class BlurNRTIndex extends BlurIndex {
   private String _table;
   private String _shard;
   private Similarity _similarity;
-  private Thread _refresher;
+  private NRTManagerReopenThread _refresher;
   private TransactionRecorder _recorder;
   private Configuration _configuration;
   private Path _walPath;
@@ -54,29 +54,34 @@ public class BlurNRTIndex extends BlurIndex {
   private BlurIndexCloser _closer;
   private AtomicReference<IndexReader> _indexRef = new AtomicReference<IndexReader>();
   private long _timeBetweenCommits = TimeUnit.SECONDS.toMillis(60);
-  private long _timeBetweenRefreshs = TimeUnit.MILLISECONDS.toMillis(500);
+  private long _timeBetweenRefreshs = TimeUnit.MILLISECONDS.toMillis(5000);
   private DirectoryReferenceFileGC _gc;
   private TrackingIndexWriter _trackingWriter;
   private SearcherFactory _searcherFactory = new SearcherFactory();
-//  private SearcherWarmer _warmer = new SearcherWarmer() {
-//    @Override
-//    public void warm(IndexSearcher s) throws IOException {
-//      IndexReader indexReader = s.getIndexReader();
-//      IndexReader[] subReaders = indexReader.getSequentialSubReaders();
-//      if (subReaders == null) {
-//        PrimeDocCache.getPrimeDocBitSet(indexReader);
-//      } else {
-//        for (IndexReader reader : subReaders) {
-//          PrimeDocCache.getPrimeDocBitSet(reader);
-//        }
-//      }
-//    }
-//  };
+  private long _lastRefresh;
+  private long _timeBetweenRefreshsNanos;
+
+  // private SearcherWarmer _warmer = new SearcherWarmer() {
+  // @Override
+  // public void warm(IndexSearcher s) throws IOException {
+  // IndexReader indexReader = s.getIndexReader();
+  // IndexReader[] subReaders = indexReader.getSequentialSubReaders();
+  // if (subReaders == null) {
+  // PrimeDocCache.getPrimeDocBitSet(indexReader);
+  // } else {
+  // for (IndexReader reader : subReaders) {
+  // PrimeDocCache.getPrimeDocBitSet(reader);
+  // }
+  // }
+  // }
+  // };
 
   public void init() throws IOException {
     Path walTablePath = new Path(_walPath, _table);
     Path walShardPath = new Path(walTablePath, _shard);
 
+    _timeBetweenRefreshsNanos = TimeUnit.MILLISECONDS.toNanos(_timeBetweenRefreshs);
+
     IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
     conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
     conf.setSimilarity(_similarity);
@@ -84,11 +89,6 @@ public class BlurNRTIndex extends BlurIndex {
     TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);
     DirectoryReferenceCounter referenceCounter = new DirectoryReferenceCounter(_directory, _gc);
-    // NRTCachingDirectory cachingDirectory = new
-    // NRTCachingDirectory(referenceCounter, _nrtCachingMaxMergeSizeMB,
-    // _nrtCachingMaxCachedMB);
-    // conf.setMergeScheduler(cachingDirectory.getMergeScheduler());
-    // _writer = new IndexWriter(cachingDirectory, conf);
     _writer = new IndexWriter(referenceCounter, conf);
     _recorder = new TransactionRecorder();
     _recorder.setAnalyzer(_analyzer);
@@ -96,16 +96,17 @@ public class BlurNRTIndex extends BlurIndex {
     _recorder.setWalPath(walShardPath);
     _recorder.init();
     _recorder.replay(_writer);
-    
+
     _trackingWriter = new TrackingIndexWriter(_writer);
     _nrtManager = new NRTManager(_trackingWriter, _searcherFactory, APPLY_ALL_DELETES);
     IndexSearcher searcher = _nrtManager.acquire();
     _indexRef.set(searcher.getIndexReader());
+    _lastRefresh = System.nanoTime();
     startCommiter();
     startRefresher();
   }
 
-  private void startRefresher() {    
+  private void startRefresher() {
     double targetMinStaleSec = _timeBetweenRefreshs / 1000.0;
     _refresher = new NRTManagerReopenThread(_nrtManager, targetMinStaleSec * 10, targetMinStaleSec);
     _refresher.setName("Refresh Thread [" + _table + "/" + _shard + "]");
@@ -174,7 +175,7 @@ public class BlurNRTIndex extends BlurIndex {
     // @TODO make sure that locks are cleaned up.
     _isClosed.set(true);
     _committer.interrupt();
-    _refresher.interrupt();
+    _refresher.close();
     try {
       _recorder.close();
       _writer.close();
@@ -188,6 +189,7 @@ public class BlurNRTIndex extends BlurIndex {
   @Override
   public void refresh() throws IOException {
     _nrtManager.maybeRefresh();
+    swap();
   }
 
   @Override
@@ -201,10 +203,16 @@ public class BlurNRTIndex extends BlurIndex {
   }
 
   private void waitToBeVisible(boolean waitToBeVisible, long generation) throws IOException {
-    if (waitToBeVisible) {
+    if (waitToBeVisible && _nrtManager.getCurrentSearchingGen() < generation) {
       // if visibility is required then reopen.
       _nrtManager.waitForGeneration(generation);
       swap();
+    } else {
+      long now = System.nanoTime();
+      if (_lastRefresh + _timeBetweenRefreshsNanos < now) {
+        refresh();
+        _lastRefresh = now;
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/DirectoryReferenceCounter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/DirectoryReferenceCounter.java b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/DirectoryReferenceCounter.java
index a4972f0..0a13b42 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/DirectoryReferenceCounter.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/manager/writer/DirectoryReferenceCounter.java
@@ -239,6 +239,7 @@ public class DirectoryReferenceCounter extends Directory {
     return directory.fileExists(name);
   }
 
+  @SuppressWarnings("deprecation")
   public long fileModified(String name) throws IOException {
     return directory.fileModified(name);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurControllerServer.java b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurControllerServer.java
index aa391e1..aac4f04 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurControllerServer.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurControllerServer.java
@@ -682,7 +682,7 @@ public class BlurControllerServer extends TableAdmin implements Iface {
 
       String shardName = MutationHelper.getShardName(table, mutation.rowId, numberOfShards, _blurPartitioner);
       String node = tableLayout.get(shardName);
-      List<RowMutation> list = batches.get(shardName);
+      List<RowMutation> list = batches.get(node);
       if (list == null) {
         list = new ArrayList<RowMutation>();
         batches.put(node, list);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurShardServer.java
index 4bb2498..c6d8195 100644
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurShardServer.java
+++ b/src/blur-core/src/main/java/com/nearinfinity/blur/thrift/BlurShardServer.java
@@ -236,10 +236,15 @@ public class BlurShardServer extends TableAdmin implements Iface {
   @Override
   public void mutateBatch(List<RowMutation> mutations) throws BlurException, TException {
     for (RowMutation mutation : mutations) {
+      checkTable(_cluster, mutation.table);
+      checkForUpdates(_cluster, mutation.table);
       MutationHelper.validateMutation(mutation);
     }
-    for (RowMutation mutation : mutations) {
-      mutate(mutation);
+    try {
+      _indexManager.mutate(mutations);
+    } catch (Exception e) {
+      LOG.error("Unknown error during processing of [mutations={0}]", e, mutations);
+      throw new BException(e.getMessage(), e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/test/java/com/nearinfinity/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/manager/IndexManagerTest.java
index bc33c47..2a22b0f 100644
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/manager/IndexManagerTest.java
+++ b/src/blur-core/src/test/java/com/nearinfinity/blur/manager/IndexManagerTest.java
@@ -636,7 +636,7 @@ public class IndexManagerTest {
     String rec = "record-6";
     RecordMutation rm = newRecordMutation(REPLACE_ENTIRE_RECORD, FAMILY, rec, c1, c2, c3);
 
-    Record r = updateAndFetchRecord("row-6", rec, rm);
+    updateAndFetchRecord("row-6", rec, rm);
   }
 
   @Test
@@ -720,7 +720,7 @@ public class IndexManagerTest {
     String rec = "record-1B";
     RecordMutation rm = newRecordMutation(REPLACE_COLUMNS, FAMILY, rec, c1, c2);
 
-    Record r = updateAndFetchRecord("row-1", rec, rm);
+    updateAndFetchRecord("row-1", rec, rm);
   }
 
   @Test(expected = BlurException.class)
@@ -730,7 +730,7 @@ public class IndexManagerTest {
     String rec = "record-6";
     RecordMutation rm = newRecordMutation(REPLACE_COLUMNS, FAMILY, rec, c1, c2);
 
-    Record r = updateAndFetchRecord("row-6", rec, rm);
+    updateAndFetchRecord("row-6", rec, rm);
   }
 
   @Test
@@ -781,7 +781,7 @@ public class IndexManagerTest {
     String rec = "record-1B";
     RecordMutation rm = newRecordMutation(APPEND_COLUMN_VALUES, FAMILY, rec, c1, c2, c3);
 
-    Record r = updateAndFetchRecord("row-1", rec, rm);
+    updateAndFetchRecord("row-1", rec, rm);
   }
 
   @Test(expected = BlurException.class)
@@ -791,7 +791,7 @@ public class IndexManagerTest {
     String rec = "record-6";
     RecordMutation rm = newRecordMutation(APPEND_COLUMN_VALUES, FAMILY, rec, c1, c2);
 
-    Record r = updateAndFetchRecord("row-6", rec, rm);
+    updateAndFetchRecord("row-6", rec, rm);
   }
 
   private Record updateAndFetchRecord(String rowId, String recordId, RecordMutation... recordMutations) throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/test/java/com/nearinfinity/blur/manager/writer/BlurNRTIndexTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/manager/writer/BlurNRTIndexTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/manager/writer/BlurNRTIndexTest.java
index 972d512..ee47a99 100644
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/manager/writer/BlurNRTIndexTest.java
+++ b/src/blur-core/src/test/java/com/nearinfinity/blur/manager/writer/BlurNRTIndexTest.java
@@ -18,7 +18,6 @@ import org.junit.Test;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
 import com.nearinfinity.blur.concurrent.Executors;
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.lucene.search.FairSimilarity;
 import com.nearinfinity.blur.thrift.generated.Column;
 import com.nearinfinity.blur.thrift.generated.Record;
@@ -42,7 +41,7 @@ public class BlurNRTIndexTest {
     base.mkdirs();
     closer = new BlurIndexCloser();
     closer.init();
-    
+
     Configuration configuration = new Configuration();
 
     BlurAnalyzer analyzer = new BlurAnalyzer(new KeywordAnalyzer());
@@ -51,18 +50,18 @@ public class BlurNRTIndexTest {
     refresher.init();
 
     writer = new BlurNRTIndex();
-    writer.setDirectory(DirectIODirectory.wrap(FSDirectory.open(new File(base,"index"))));
+    writer.setDirectory(FSDirectory.open(new File(base, "index")));
     writer.setCloser(closer);
     writer.setAnalyzer(analyzer);
     writer.setSimilarity(new FairSimilarity());
     writer.setTable("testing-table");
     writer.setShard("testing-shard");
-    
+
     service = Executors.newThreadPool("test", 10);
-    writer.setWalPath(new Path(new File(base,"wal").toURI()));
-    
+    writer.setWalPath(new Path(new File(base, "wal").toURI()));
+
     writer.setConfiguration(configuration);
-    writer.setTimeBetweenRefreshs(2000);
+    writer.setTimeBetweenRefreshs(25);
     writer.init();
   }
 
@@ -94,7 +93,7 @@ public class BlurNRTIndexTest {
     for (int i = 0; i < TEST_NUMBER_WAIT_VISIBLE; i++) {
       writer.replaceRow(true, true, genRow());
       IndexReader reader = writer.getIndexReader();
-      assertEquals(i, reader.numDocs());
+      assertEquals(i + 1, reader.numDocs());
       total++;
     }
     long e = System.nanoTime();
@@ -104,7 +103,7 @@ public class BlurNRTIndexTest {
     IndexReader reader = writer.getIndexReader();
     assertEquals(TEST_NUMBER_WAIT_VISIBLE, reader.numDocs());
   }
-  
+
   @Test
   public void testBlurIndexWriterFaster() throws IOException, InterruptedException {
     long s = System.nanoTime();
@@ -117,7 +116,7 @@ public class BlurNRTIndexTest {
     double seconds = (e - s) / 1000000000.0;
     double rate = total / seconds;
     System.out.println("Rate " + rate);
-    Thread.sleep(5000);
+    writer.refresh();
     IndexReader reader = writer.getIndexReader();
     assertEquals(TEST_NUMBER, reader.numDocs());
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/test/java/com/nearinfinity/blur/search/RandomSuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/search/RandomSuperQueryTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/search/RandomSuperQueryTest.java
index 37dc8a9..d94604a 100644
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/search/RandomSuperQueryTest.java
+++ b/src/blur-core/src/test/java/com/nearinfinity/blur/search/RandomSuperQueryTest.java
@@ -43,7 +43,6 @@ import org.apache.lucene.store.RAMDirectory;
 import org.junit.Test;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.index.IndexWriter;
 import com.nearinfinity.blur.lucene.search.SuperParser;
 import com.nearinfinity.blur.thrift.generated.Column;
@@ -98,7 +97,7 @@ public class RandomSuperQueryTest {
     for (int i = 0; i < columnFamilies.length; i++) {
       columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col"));
     }
-    IndexWriter writer = new IndexWriter(DirectIODirectory.wrap(directory), new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
+    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
     RowIndexWriter indexWriter = new RowIndexWriter(writer, new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION)));
     int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1;
     for (int i = 0; i < numberOfDocs; i++) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/test/java/com/nearinfinity/blur/search/SuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/search/SuperQueryTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/search/SuperQueryTest.java
index 7a8008a..0765155 100644
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/search/SuperQueryTest.java
+++ b/src/blur-core/src/test/java/com/nearinfinity/blur/search/SuperQueryTest.java
@@ -47,7 +47,6 @@ import org.apache.lucene.store.RAMDirectory;
 import org.junit.Test;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.index.IndexWriter;
 import com.nearinfinity.blur.lucene.search.FacetQuery;
 import com.nearinfinity.blur.lucene.search.SuperQuery;
@@ -158,10 +157,10 @@ public class SuperQueryTest {
     booleanQuery.add(wrapSuper(ADDRESS_STREET, STREET1, ScoreType.SUPER), Occur.MUST);
 
     BooleanQuery f1 = new BooleanQuery();
-    f1.add(new TermQuery(new Term(PERSON_NAME,NAME1)),Occur.MUST);
-    f1.add(new TermQuery(new Term(PERSON_NAME,NAME2)),Occur.MUST);
-    
-    Query[] facets = new Query[] {new SuperQuery(f1, ScoreType.CONSTANT)};
+    f1.add(new TermQuery(new Term(PERSON_NAME, NAME1)), Occur.MUST);
+    f1.add(new TermQuery(new Term(PERSON_NAME, NAME2)), Occur.MUST);
+
+    Query[] facets = new Query[] { new SuperQuery(f1, ScoreType.CONSTANT) };
     AtomicLongArray counts = new AtomicLongArray(facets.length);
     FacetQuery query = new FacetQuery(booleanQuery, facets, counts);
 
@@ -188,7 +187,7 @@ public class SuperQueryTest {
 
   public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
     Directory directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(DirectIODirectory.wrap(directory), new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
+    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
     BlurAnalyzer analyzer = new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION));
     RowIndexWriter indexWriter = new RowIndexWriter(writer, analyzer);
     indexWriter.replace(

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-core/src/test/java/com/nearinfinity/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/thrift/BlurClusterTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/thrift/BlurClusterTest.java
index 7cca0cf..1192921 100644
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/thrift/BlurClusterTest.java
+++ b/src/blur-core/src/test/java/com/nearinfinity/blur/thrift/BlurClusterTest.java
@@ -3,6 +3,7 @@ package com.nearinfinity.blur.thrift;
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.UUID;
@@ -56,15 +57,15 @@ public class BlurClusterTest {
   public void testLoadTable() throws BlurException, TException, InterruptedException {
     Iface client = getClient();
     int length = 100;
+    List<RowMutation> mutations = new ArrayList<RowMutation>();
     for (int i = 0; i < length; i++) {
       String rowId = UUID.randomUUID().toString();
       RecordMutation mutation = BlurUtil.newRecordMutation("test", rowId, BlurUtil.newColumn("test", "value"));
       RowMutation rowMutation = BlurUtil.newRowMutation("test", rowId, mutation);
-      if (i == length - 1) {
-        rowMutation.setWaitToBeVisible(true);
-      }
-      client.mutate(rowMutation);
+      rowMutation.setWaitToBeVisible(true);
+      mutations.add(rowMutation);
     }
+    client.mutateBatch(mutations);
     BlurQuery blurQuery = new BlurQuery();
     SimpleQuery simpleQuery = new SimpleQuery();
     simpleQuery.setQueryStr("test.test:value");

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BlurReducer.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BlurReducer.java b/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BlurReducer.java
index 9050ffc..3c6975e 100644
--- a/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BlurReducer.java
+++ b/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BlurReducer.java
@@ -61,7 +61,6 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.zookeeper.ZooKeeper;
 
 import com.nearinfinity.blur.analysis.BlurAnalyzer;
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.log.Log;
 import com.nearinfinity.blur.log.LogFactory;
 import com.nearinfinity.blur.lucene.search.FairSimilarity;
@@ -137,7 +136,7 @@ public class BlurReducer extends Reducer<BytesWritable, BlurMutate, BytesWritabl
     setupDirectory(context);
     setupWriter(context);
     if (_blurTask.getIndexingType() == INDEXING_TYPE.UPDATE) {
-      _reader = IndexReader.open(_directory, true);
+      _reader = IndexReader.open(_directory);
     }
   }
 
@@ -189,7 +188,7 @@ public class BlurReducer extends Reducer<BytesWritable, BlurMutate, BytesWritabl
     }
 
     List<Document> docs = documentsToIndex(new ArrayList<Document>(_newDocs.values()));
-    if (docs.size() >0) {
+    if (docs.size() > 0) {
       docs.get(0).add(BlurConstants.PRIME_DOC_FIELD);
     }
 
@@ -432,8 +431,7 @@ public class BlurReducer extends Reducer<BytesWritable, BlurMutate, BytesWritabl
       // if (compressionBlockSize == 0) {
       compressionBlockSize = 32768;
       // }
-      CompressedFieldDataDirectory compressedFieldDataDirectory = new CompressedFieldDataDirectory(DirectIODirectory.wrap(localDirectory), getInstance(compressionClass),
-          compressionBlockSize);
+      CompressedFieldDataDirectory compressedFieldDataDirectory = new CompressedFieldDataDirectory(localDirectory, getInstance(compressionClass), compressionBlockSize);
       _directory = new ProgressableDirectory(compressedFieldDataDirectory, context);
       return;
     default:
@@ -500,8 +498,8 @@ public class BlurReducer extends Reducer<BytesWritable, BlurMutate, BytesWritabl
     double rate = totalBytesCopied / seconds;
     String time = estimateTimeToComplete(rate, totalBytesCopied, totalBytesToCopy);
 
-    String status = String.format("%.1f Complete - Time Remaining [%s s], Copy rate [%.1f MB/s], Total Copied [%.1f MB], Total To Copy [%.1f MB]", getPerComplete(totalBytesCopied,
-        totalBytesToCopy), time, getMb(rate), getMb(totalBytesCopied), getMb(totalBytesToCopy));
+    String status = String.format("%.1f Complete - Time Remaining [%s s], Copy rate [%.1f MB/s], Total Copied [%.1f MB], Total To Copy [%.1f MB]",
+        getPerComplete(totalBytesCopied, totalBytesToCopy), time, getMb(rate), getMb(totalBytesCopied), getMb(totalBytesToCopy));
     LOG.info(status);
     context.setStatus(status);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BufferedDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BufferedDirectory.java b/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BufferedDirectory.java
index 93cb8f7..5a5cdce 100644
--- a/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BufferedDirectory.java
+++ b/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/BufferedDirectory.java
@@ -40,6 +40,7 @@ public class BufferedDirectory extends Directory {
     return _directory.fileLength(name);
   }
 
+  @SuppressWarnings("deprecation")
   public long fileModified(String name) throws IOException {
     return _directory.fileModified(name);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/ProgressableDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/ProgressableDirectory.java b/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/ProgressableDirectory.java
index b429b3d..72169c3 100644
--- a/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/ProgressableDirectory.java
+++ b/src/blur-mapred/src/main/java/com/nearinfinity/blur/mapreduce/ProgressableDirectory.java
@@ -63,6 +63,7 @@ public class ProgressableDirectory extends Directory {
     return _directory.fileLength(name);
   }
 
+  @SuppressWarnings("deprecation")
   public long fileModified(String name) throws IOException {
     return _directory.fileModified(name);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/main/java/com/nearinfinity/blur/index/DirectIODirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/index/DirectIODirectory.java b/src/blur-store/src/main/java/com/nearinfinity/blur/index/DirectIODirectory.java
deleted file mode 100644
index ec76dd0..0000000
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/index/DirectIODirectory.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package com.nearinfinity.blur.index;
-
-import java.io.IOException;
-
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-
-public abstract class DirectIODirectory extends Directory {
-  
-  public static final int BUFFER_SIZE = 8192;
-  
-  public abstract IndexOutput createOutputDirectIO(String name) throws IOException;
-
-  public abstract IndexInput openInputDirectIO(String name) throws IOException;
-
-  public static DirectIODirectory wrap(Directory dir) throws IOException {
-    return new DirectIODirectoryWrapper(dir);
-  }
-
-  public static class DirectIODirectoryWrapper extends DirectIODirectory {
-    
-    private Directory _directory;
-
-    public DirectIODirectoryWrapper(Directory directory) throws IOException {
-      _directory = directory;
-      setLockFactory(directory.getLockFactory());
-    }
-
-    public void close() throws IOException {
-      _directory.close();
-    }
-
-    public IndexOutput createOutput(String arg0) throws IOException {
-      return _directory.createOutput(arg0);
-    }
-
-    public void deleteFile(String arg0) throws IOException {
-      _directory.deleteFile(arg0);
-    }
-
-    public boolean fileExists(String arg0) throws IOException {
-      return _directory.fileExists(arg0);
-    }
-
-    public long fileLength(String arg0) throws IOException {
-      return _directory.fileLength(arg0);
-    }
-
-    public long fileModified(String arg0) throws IOException {
-      return _directory.fileModified(arg0);
-    }
-
-    public String[] listAll() throws IOException {
-      return _directory.listAll();
-    }
-
-    public IndexInput openInput(String arg0) throws IOException {
-      return _directory.openInput(arg0);
-    }
-
-    @SuppressWarnings("deprecation")
-    public void touchFile(String arg0) throws IOException {
-      _directory.touchFile(arg0);
-    }
-
-    @Override
-    public IndexOutput createOutputDirectIO(String name) throws IOException {
-      return createOutput(name);
-    }
-
-    @Override
-    public IndexInput openInputDirectIO(String name) throws IOException {
-      return openInput(name);
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java b/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
index 47c581c..6a5779c 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/lucene/EscapeRewrite.java
@@ -2,7 +2,6 @@ package com.nearinfinity.blur.lucene;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -14,7 +13,6 @@ import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.StaleReaderException;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.TermEnum;
@@ -27,8 +25,8 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.LockObtainFailedException;
 
+@SuppressWarnings("deprecation")
 public class EscapeRewrite {
 
   public static void main(String[] args) throws CorruptIndexException, IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
index 46165fc..d86d229 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/blockcache/BlockDirectory.java
@@ -10,11 +10,10 @@ import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockFactory;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.store.BufferStore;
 import com.nearinfinity.blur.store.CustomBufferedIndexInput;
 
-public class BlockDirectory extends DirectIODirectory {
+public class BlockDirectory extends Directory {
 
   public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
   public static final long BLOCK_MOD = 0x1FFF;
@@ -55,21 +54,21 @@ public class BlockDirectory extends DirectIODirectory {
     }
   };
 
-  private DirectIODirectory _directory;
+  private Directory _directory;
   private int _blockSize;
   private String _dirName;
   private Cache _cache;
   private Set<String> _blockCacheFileTypes;
 
-  public BlockDirectory(String dirName, DirectIODirectory directory) throws IOException {
+  public BlockDirectory(String dirName, Directory directory) throws IOException {
     this(dirName, directory, NO_CACHE);
   }
 
-  public BlockDirectory(String dirName, DirectIODirectory directory, Cache cache) throws IOException {
+  public BlockDirectory(String dirName, Directory directory, Cache cache) throws IOException {
     this(dirName, directory, cache, null);
   }
 
-  public BlockDirectory(String dirName, DirectIODirectory directory, Cache cache, Set<String> blockCacheFileTypes) throws IOException {
+  public BlockDirectory(String dirName, Directory directory, Cache cache, Set<String> blockCacheFileTypes) throws IOException {
     _dirName = dirName;
     _directory = directory;
     _blockSize = BLOCK_SIZE;
@@ -250,6 +249,7 @@ public class BlockDirectory extends DirectIODirectory {
     return _directory.fileLength(name);
   }
 
+  @SuppressWarnings("deprecation")
   public long fileModified(String name) throws IOException {
     return _directory.fileModified(name);
   }
@@ -263,14 +263,4 @@ public class BlockDirectory extends DirectIODirectory {
     _directory.touchFile(name);
   }
 
-  @Override
-  public IndexOutput createOutputDirectIO(String name) throws IOException {
-    return _directory.createOutputDirectIO(name);
-  }
-
-  @Override
-  public IndexInput openInputDirectIO(String name) throws IOException {
-    return _directory.openInputDirectIO(name);
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/main/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectory.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectory.java
index b7e65d5..503d018 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectory.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectory.java
@@ -30,11 +30,10 @@ import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockFactory;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.log.Log;
 import com.nearinfinity.blur.log.LogFactory;
 
-public class CompressedFieldDataDirectory extends DirectIODirectory {
+public class CompressedFieldDataDirectory extends Directory {
 
   private static final Log LOG = LogFactory.getLog(CompressedFieldDataDirectory.class);
 
@@ -48,22 +47,22 @@ public class CompressedFieldDataDirectory extends DirectIODirectory {
   public static CompressionCodec DEFAULT_COMPRESSION = new DefaultCodec();
 
   private CompressionCodec _compression = DEFAULT_COMPRESSION;
-  private DirectIODirectory _directory;
+  private Directory _directory;
   private int _writingBlockSize;
 
   public Directory getInnerDirectory() {
     return _directory;
   }
 
-  public CompressedFieldDataDirectory(DirectIODirectory dir) {
+  public CompressedFieldDataDirectory(Directory dir) {
     this(dir, DEFAULT_COMPRESSION);
   }
 
-  public CompressedFieldDataDirectory(DirectIODirectory dir, CompressionCodec compression) {
+  public CompressedFieldDataDirectory(Directory dir, CompressionCodec compression) {
     this(dir, compression, COMPRESSED_BUFFER_SIZE);
   }
 
-  public CompressedFieldDataDirectory(DirectIODirectory dir, CompressionCodec compression, int blockSize) {
+  public CompressedFieldDataDirectory(Directory dir, CompressionCodec compression, int blockSize) {
     _directory = dir;
     if (compression == null) {
       _compression = DEFAULT_COMPRESSION;
@@ -182,6 +181,7 @@ public class CompressedFieldDataDirectory extends DirectIODirectory {
     return _directory.fileLength(name);
   }
 
+  @SuppressWarnings("deprecation")
   public long fileModified(String name) throws IOException {
     if (compressedFileExists(name)) {
       return _directory.fileModified(getCompressedName(name));
@@ -812,14 +812,4 @@ public class CompressedFieldDataDirectory extends DirectIODirectory {
       _pos = pos;
     }
   }
-
-  @Override
-  public IndexOutput createOutputDirectIO(String name) throws IOException {
-    return _directory.createOutputDirectIO(name);
-  }
-
-  @Override
-  public IndexInput openInputDirectIO(String name) throws IOException {
-    return _directory.openInputDirectIO(name);
-  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsDirectory.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsDirectory.java
index 3b7b933..114d9e1 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsDirectory.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsDirectory.java
@@ -24,19 +24,19 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.store.CustomBufferedIndexInput;
 
-/** @author Aaron McCurry (amccurry@nearinfinity.com) */
-public class HdfsDirectory extends DirectIODirectory {
+public class HdfsDirectory extends Directory {
+  
+  public static final int BUFFER_SIZE = 8192;
 
   private static final String LF_EXT = ".lf";
   protected static final String SEGMENTS_GEN = "segments.gen";
@@ -322,66 +322,6 @@ public class HdfsDirectory extends DirectIODirectory {
     }
   }
 
-  @Override
-  public IndexOutput createOutputDirectIO(String name) throws IOException {
-    FileSystem fileSystem = getFileSystem();
-    final FSDataOutputStream outputStream = fileSystem.create(new Path(_hdfsDirPath, name));
-    return new IndexOutput() {
-
-      @Override
-      public void close() throws IOException {
-        outputStream.close();
-      }
-
-      @Override
-      public void flush() throws IOException {
-        outputStream.sync();
-      }
-
-      @Override
-      public long getFilePointer() {
-        try {
-          return outputStream.getPos();
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-      }
-
-      @Override
-      public long length() throws IOException {
-        return outputStream.getPos();
-      }
-
-      @Override
-      public void seek(long pos) throws IOException {
-        throw new IOException("Seeks not allowed");
-      }
-
-      @Override
-      public void writeByte(byte b) throws IOException {
-        outputStream.write(b & 0xFF);
-      }
-
-      @Override
-      public void writeBytes(byte[] b, int offset, int length) throws IOException {
-        outputStream.write(b, offset, length);
-      }
-    };
-  }
-
-  @Override
-  public IndexInput openInputDirectIO(String name) throws IOException {
-    Path path = new Path(_hdfsDirPath, name);
-    FSDataInputStream inputStream = getFileSystem().open(path);
-    return new DirectIOHdfsIndexInput(name, inputStream, realFileLength(path));
-  }
-
-  private long realFileLength(Path path) throws IOException {
-    FileSystem fileSystem = getFileSystem();
-    FileStatus fileStatus = fileSystem.getFileStatus(path);
-    return fileStatus.getLen();
-  }
-
   static class DirectIOHdfsIndexInput extends CustomBufferedIndexInput {
 
     private long _length;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsFileReader.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsFileReader.java b/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsFileReader.java
index 00c1472..bdbe653 100644
--- a/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsFileReader.java
+++ b/src/blur-store/src/main/java/com/nearinfinity/blur/store/hdfs/HdfsFileReader.java
@@ -11,12 +11,11 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.store.DataInput;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.log.Log;
 import com.nearinfinity.blur.log.LogFactory;
 
 public class HdfsFileReader extends DataInput {
-  
+
   private static final Log LOG = LogFactory.getLog(HdfsFileReader.class);
 
   private static final int VERSION = -1;
@@ -36,7 +35,7 @@ public class HdfsFileReader extends DataInput {
     }
     FileStatus fileStatus = fileSystem.getFileStatus(path);
     _hdfsLength = fileStatus.getLen();
-    _inputStream = fileSystem.open(path,bufferSize);
+    _inputStream = fileSystem.open(path, bufferSize);
 
     // read meta blocks
     _inputStream.seek(_hdfsLength - 16);
@@ -56,9 +55,9 @@ public class HdfsFileReader extends DataInput {
     }
     seek(0);
   }
-  
+
   public HdfsFileReader(FileSystem fileSystem, Path path) throws IOException {
-    this(fileSystem,path,DirectIODirectory.BUFFER_SIZE);
+    this(fileSystem, path, HdfsDirectory.BUFFER_SIZE);
   }
 
   public long getPosition() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
index 93f7d41..91c28f1 100644
--- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
+++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectory.java
@@ -28,21 +28,21 @@ import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.WildcardQuery;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.metrics.BlurMetrics;
 import com.nearinfinity.blur.store.blockcache.BlockCache;
 import com.nearinfinity.blur.store.blockcache.BlockDirectory;
 import com.nearinfinity.blur.store.blockcache.BlockDirectoryCache;
 import com.nearinfinity.blur.store.hdfs.HdfsDirectory;
 import com.nearinfinity.blur.store.lock.BlurLockFactory;
+
 public class BenchmarkDirectory {
 
   public static void main(String[] args) throws IOException {
     int blockSize = BlockDirectory.BLOCK_SIZE;
     long totalMemory = BlockCache._128M * 2;
     int slabSize = (int) (totalMemory / 2);
-    
-    BlockCache blockCache = new BlockCache(new BlurMetrics(new Configuration()),true,totalMemory,slabSize,blockSize);
+
+    BlockCache blockCache = new BlockCache(new BlurMetrics(new Configuration()), true, totalMemory, slabSize, blockSize);
     BlurMetrics metrics = new BlurMetrics(new Configuration());
     BlockDirectoryCache cache = new BlockDirectoryCache(blockCache, metrics);
 
@@ -50,14 +50,13 @@ public class BenchmarkDirectory {
     Path p = new Path("hdfs://localhost:9000/bench");
     BlurLockFactory factory = new BlurLockFactory(configuration, p, "localhost", 0);
 
-    
     FileSystem fs = FileSystem.get(p.toUri(), configuration);
     fs.delete(p, true);
 
     final HdfsDirectory dir = new HdfsDirectory(p);
     dir.setLockFactory(factory);
-    
-    BlockDirectory directory = new BlockDirectory("test", DirectIODirectory.wrap(dir), cache);
+
+    BlockDirectory directory = new BlockDirectory("test", dir, cache);
 
     while (true) {
       long s, e;
@@ -80,8 +79,7 @@ public class BenchmarkDirectory {
       List<Term> sample = new ArrayList<Term>();
       int limit = 1000;
       Random random = new Random();
-      SAMPLE:
-      while (terms.next()) {
+      SAMPLE: while (terms.next()) {
         if (sample.size() < limit) {
           if (random.nextInt() % 7 == 0) {
             sample.add(terms.term());
@@ -104,7 +102,7 @@ public class BenchmarkDirectory {
         e = System.currentTimeMillis();
         time += (e - s);
       }
-      System.out.println("Searching " + time + " " + (time / (double) search));
+      System.out.println("Searching " + time + " " + (time / (double) search) + " " + total);
       for (int i = 0; i < 10; i++) {
         s = System.currentTimeMillis();
         TopDocs topDocs = searcher.search(new WildcardQuery(new Term("name", "fff*0*")), 10);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
index 72552ac..7573e8b 100644
--- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
+++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/BenchmarkDirectoryNrt.java
@@ -26,20 +26,20 @@ import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.store.NoLockFactory;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 import com.nearinfinity.blur.metrics.BlurMetrics;
 import com.nearinfinity.blur.store.blockcache.BlockCache;
 import com.nearinfinity.blur.store.blockcache.BlockDirectory;
 import com.nearinfinity.blur.store.blockcache.BlockDirectoryCache;
 import com.nearinfinity.blur.store.hdfs.HdfsDirectory;
+
 public class BenchmarkDirectoryNrt {
 
   public static void main(String[] args) throws IOException, InterruptedException {
     int blockSize = BlockDirectory.BLOCK_SIZE;
     long totalMemory = BlockCache._128M * 2;
     int slabSize = (int) (totalMemory / 2);
-    
-    BlockCache blockCache = new BlockCache(new BlurMetrics(new Configuration()),true,totalMemory,slabSize,blockSize);
+
+    BlockCache blockCache = new BlockCache(new BlurMetrics(new Configuration()), true, totalMemory, slabSize, blockSize);
     BlurMetrics metrics = new BlurMetrics(new Configuration());
     BlockDirectoryCache cache = new BlockDirectoryCache(blockCache, metrics);
 
@@ -50,7 +50,7 @@ public class BenchmarkDirectoryNrt {
     final HdfsDirectory dir = new HdfsDirectory(p);
     dir.setLockFactory(NoLockFactory.getNoLockFactory());
 
-    BlockDirectory directory = new BlockDirectory("test", DirectIODirectory.wrap(dir), cache);
+    BlockDirectory directory = new BlockDirectory("test", dir, cache);
 
     while (true) {
       IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION));
@@ -62,16 +62,15 @@ public class BenchmarkDirectoryNrt {
       IndexWriter writer = new IndexWriter(directory, conf);
       long as = System.currentTimeMillis();
       BlockingQueue<Collection<Document>> queue = new ArrayBlockingQueue<Collection<Document>>(1024);
-      Indexer indexer = new Indexer(queue,writer);
+      Indexer indexer = new Indexer(queue, writer);
       new Thread(indexer).start();
       for (int i = 0; i < 1000000; i++) {
         if (count >= max) {
-          double aseconds = (System.currentTimeMillis()-as) / 1000.0;
+          double aseconds = (System.currentTimeMillis() - as) / 1000.0;
           double arate = i / aseconds;
-          double seconds = (System.currentTimeMillis()-s) / 1000.0;
+          double seconds = (System.currentTimeMillis() - s) / 1000.0;
           double rate = count / seconds;
-          System.out.println("Total [" + i + "] Rate [" + rate + "] AvgRate [" + arate +
-          		"] Doc count [" + indexer.getReader().numDocs() + "]");
+          System.out.println("Total [" + i + "] Rate [" + rate + "] AvgRate [" + arate + "] Doc count [" + indexer.getReader().numDocs() + "]");
           count = 0;
           s = System.currentTimeMillis();
         }
@@ -81,9 +80,9 @@ public class BenchmarkDirectoryNrt {
       writer.close();
     }
   }
-  
+
   private static class Indexer implements Runnable {
-    
+
     private BlockingQueue<Collection<Document>> _queue;
     private AtomicBoolean _running = new AtomicBoolean(true);
     private IndexWriter _writer;
@@ -94,7 +93,7 @@ public class BenchmarkDirectoryNrt {
       _writer = writer;
       _reader = IndexReader.open(_writer, true);
     }
-    
+
     public IndexReader getReader() {
       return _reader;
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/test/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryTest.java b/src/blur-store/src/test/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryTest.java
index 567436a..8a65968 100644
--- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryTest.java
+++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/blockcache/BlockDirectoryTest.java
@@ -16,7 +16,6 @@ import org.junit.Before;
 import org.junit.Test;
 
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.nearinfinity.blur.index.DirectIODirectory;
 
 public class BlockDirectoryTest {
 
@@ -37,7 +36,7 @@ public class BlockDirectoryTest {
     rm(file);
     file.mkdirs();
     FSDirectory dir = FSDirectory.open(new File(file, "base"));
-    directory = new BlockDirectory("test", DirectIODirectory.wrap(dir), getBasicCache());
+    directory = new BlockDirectory("test", dir, getBasicCache());
     seed = new Random().nextLong();
     random = new Random(seed);
   }
@@ -70,7 +69,7 @@ public class BlockDirectoryTest {
 
       @Override
       public void delete(String name) {
-        
+
       }
 
       @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b8476524/src/blur-store/src/test/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectoryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/test/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectoryTest.java b/src/blur-store/src/test/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectoryTest.java
index 2ea2623..60a23f9 100644
--- a/src/blur-store/src/test/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectoryTest.java
+++ b/src/blur-store/src/test/java/com/nearinfinity/blur/store/compressed/CompressedFieldDataDirectoryTest.java
@@ -22,7 +22,6 @@ import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.RAMDirectory;
 import org.junit.Test;
 
-import com.nearinfinity.blur.index.DirectIODirectory;
 public class CompressedFieldDataDirectoryTest {
 
   private static final CompressionCodec COMPRESSION_CODEC = CompressedFieldDataDirectory.DEFAULT_COMPRESSION;
@@ -30,7 +29,7 @@ public class CompressedFieldDataDirectoryTest {
   @Test
   public void testCompressedFieldDataDirectoryBasic() throws CorruptIndexException, IOException {
     RAMDirectory dir = new RAMDirectory();
-    CompressedFieldDataDirectory directory = new CompressedFieldDataDirectory(DirectIODirectory.wrap(dir), COMPRESSION_CODEC);
+    CompressedFieldDataDirectory directory = new CompressedFieldDataDirectory(dir, COMPRESSION_CODEC);
     IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
     TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);
@@ -52,7 +51,7 @@ public class CompressedFieldDataDirectoryTest {
     addDocs(writer, 0, 5);
     writer.close();
 
-    CompressedFieldDataDirectory directory = new CompressedFieldDataDirectory(DirectIODirectory.wrap(dir), COMPRESSION_CODEC);
+    CompressedFieldDataDirectory directory = new CompressedFieldDataDirectory(dir, COMPRESSION_CODEC);
     config = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
     mergePolicy = (TieredMergePolicy) config.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);
@@ -72,7 +71,7 @@ public class CompressedFieldDataDirectoryTest {
     addDocs(writer, 0, 5);
     writer.close();
 
-    CompressedFieldDataDirectory directory1 = new CompressedFieldDataDirectory(DirectIODirectory.wrap(dir), COMPRESSION_CODEC, 2);
+    CompressedFieldDataDirectory directory1 = new CompressedFieldDataDirectory(dir, COMPRESSION_CODEC, 2);
     config = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
     mergePolicy = (TieredMergePolicy) config.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);
@@ -80,7 +79,7 @@ public class CompressedFieldDataDirectoryTest {
     addDocs(writer, 5, 2);
     writer.close();
 
-    CompressedFieldDataDirectory directory2 = new CompressedFieldDataDirectory(DirectIODirectory.wrap(dir), COMPRESSION_CODEC, 4);
+    CompressedFieldDataDirectory directory2 = new CompressedFieldDataDirectory(dir, COMPRESSION_CODEC, 4);
     config = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
     mergePolicy = (TieredMergePolicy) config.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);
@@ -95,10 +94,10 @@ public class CompressedFieldDataDirectoryTest {
     String[] listAll = dir.listAll();
     for (String name : listAll) {
       IndexInput input = dir.openInput(name);
-      assertEquals(input.length(),dir.fileLength(name));
+      assertEquals(input.length(), dir.fileLength(name));
       input.close();
     }
-    
+
   }
 
   private void testFetches(Directory directory) throws CorruptIndexException, IOException {


Mime
View raw message