incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Improved the warmup process to be more efficient.
Date Sat, 28 Dec 2013 01:49:18 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 c8a56fd8b -> 0e6db78b3


Improved the warmup process to be more efficient.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e6db78b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e6db78b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e6db78b

Branch: refs/heads/apache-blur-0.2
Commit: 0e6db78b39f998334a10d71d2424fff04911b8e1
Parents: c8a56fd
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Dec 27 20:48:24 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Dec 27 20:48:24 2013 -0500

----------------------------------------------------------------------
 .../manager/indexserver/BlurIndexWarmup.java    |  34 ++++
 .../indexserver/DefaultBlurIndexWarmup.java     |  25 ++-
 .../indexserver/DistributedIndexServer.java     |   2 +-
 .../manager/indexserver/LocalIndexServer.java   |   5 +-
 .../apache/blur/manager/writer/BlurIndex.java   |   4 +-
 .../blur/manager/writer/BlurIndexReadOnly.java  |   2 +-
 .../blur/manager/writer/BlurIndexReader.java    |   7 +-
 .../manager/writer/BlurIndexReaderWarmer.java   |  60 ++++++
 .../manager/writer/BlurIndexSimpleWriter.java   |   8 +-
 .../blur/manager/writer/BlurNRTIndex.java       |   8 +-
 .../blur/manager/writer/FieldBasedWarmer.java   |  77 --------
 .../org/apache/blur/server/TableContext.java    |   9 +-
 .../blur/thrift/ThriftBlurShardServer.java      |  26 +--
 .../indexserver/DefaultBlurIndexWarmupTest.java | 162 +++++++++++++++
 .../manager/writer/BlurIndexNRTSimpleTest.java  | 124 ++++++------
 .../manager/writer/BlurIndexReaderTest.java     |   5 +-
 .../blur/manager/writer/BlurNRTIndexTest.java   |   6 +-
 .../apache/blur/lucene/warmup/IndexWarmup.java  | 198 ++++++++++++++++++-
 .../blur/lucene/warmup/ThrottledIndexInput.java |  19 +-
 .../blur/lucene/warmup/IndexWarmupTest.java     | 183 +++++++++++++++++
 .../blur/lucene/warmup/SlowAccessDirectory.java | 149 ++++++++++++++
 21 files changed, 919 insertions(+), 194 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/indexserver/BlurIndexWarmup.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/BlurIndexWarmup.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/BlurIndexWarmup.java
index 95bd19a..2bd394b 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/BlurIndexWarmup.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/BlurIndexWarmup.java
@@ -16,22 +16,56 @@ package org.apache.blur.manager.indexserver;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_THROTTLE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT;
+
 import java.io.IOException;
+import java.lang.reflect.Constructor;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.indexserver.DistributedIndexServer.ReleaseReader;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.lucene.index.IndexReader;
 
 public abstract class BlurIndexWarmup {
 
+  private static final Log LOG = LogFactory.getLog(BlurIndexWarmup.class);
+
   protected long _warmupBandwidthThrottleBytesPerSec;
 
   public BlurIndexWarmup(long warmupBandwidthThrottleBytesPerSec) {
     _warmupBandwidthThrottleBytesPerSec = warmupBandwidthThrottleBytesPerSec;
   }
 
+  public static BlurIndexWarmup getIndexWarmup(BlurConfiguration configuration) {
+    long totalThrottle = configuration.getLong(BLUR_SHARD_INDEX_WARMUP_THROTTLE, 30000000);
+    int totalThreadCount = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 30000000);
+    long warmupBandwidthThrottleBytesPerSec = totalThrottle / totalThreadCount;
+    if (warmupBandwidthThrottleBytesPerSec <= 0) {
+      LOG.warn("Invalid values of either [{0} = {1}] or [{2} = {3}], needs to be greater then 0",
+          BLUR_SHARD_INDEX_WARMUP_THROTTLE, totalThrottle, BLUR_SHARD_WARMUP_THREAD_COUNT, totalThreadCount);
+    }
+    
+    String blurFilterCacheClass = configuration.get(BLUR_SHARD_INDEX_WARMUP_CLASS);
+    if (blurFilterCacheClass != null && blurFilterCacheClass.isEmpty()) {
+      if (!blurFilterCacheClass.equals("org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup")) {
+        try {
+          Class<?> clazz = Class.forName(blurFilterCacheClass);
+          Constructor<?> constructor = clazz.getConstructor(new Class[]{Long.TYPE});
+          return (BlurIndexWarmup) constructor.newInstance(warmupBandwidthThrottleBytesPerSec);
+        } catch (Exception e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+    return new DefaultBlurIndexWarmup(warmupBandwidthThrottleBytesPerSec);
+  }
+
   /**
    * Once the reader has be warmed up, release() must be called on the
    * ReleaseReader even if an exception occurs.

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
index 2320c16..b8a0dde 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
@@ -19,6 +19,7 @@ package org.apache.blur.manager.indexserver;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -42,6 +43,7 @@ import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.util.OpenBitSet;
 
 public class DefaultBlurIndexWarmup extends BlurIndexWarmup {
 
@@ -67,6 +69,7 @@ public class DefaultBlurIndexWarmup extends BlurIndexWarmup {
   public void warmBlurIndex(final TableDescriptor table, final String shard, IndexReader reader,
       AtomicBoolean isClosed, ReleaseReader releaseReader, AtomicLong pauseWarmup) throws IOException {
     LOG.info("Running warmup for reader [{0}]", reader);
+    long s = System.nanoTime();
     AtomicBoolean stop = new AtomicBoolean();
     try {
       if (reader instanceof FilterDirectoryReader) {
@@ -79,17 +82,31 @@ public class DefaultBlurIndexWarmup extends BlurIndexWarmup {
       IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, _warmupBandwidthThrottleBytesPerSec);
       String context = table.getName() + "/" + shard;
       Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(reader, context);
-      List<String> preCacheCols = table.getPreCacheCols();
-      if (preCacheCols != null) {
+      Iterable<String> preCacheCols = table.getPreCacheCols();
+      if (preCacheCols == null) {
+        // All fields.
+        preCacheCols = getFields(reader);
+      }
+      boolean _oldWay = false;
+      if (_oldWay) {
         warm(reader, preCacheCols, indexWarmup, sampleIndex, context, isClosed, pauseWarmup);
       } else {
-        warm(reader, getFields(reader), indexWarmup, sampleIndex, context, isClosed, pauseWarmup);
+        int blockSize = 8192;// @TODO
+        int bufferSize = 1024 * 1024;// @TODO
+        Map<String, OpenBitSet> filePartsToWarm = new HashMap<String, OpenBitSet>();
+        for (String fieldName : preCacheCols) {
+          indexWarmup.getFilePositionsToWarm(reader, sampleIndex, fieldName, context, filePartsToWarm, blockSize);
+        }
+        indexWarmup.warmFile(reader, filePartsToWarm, context, blockSize, bufferSize);
       }
+
     } finally {
       synchronized (_stops) {
         _stops.remove(stop);
       }
       releaseReader.release();
+      long e = System.nanoTime();
+      LOG.info("Warmup completed for table [{0}] shard [{1}] in [{2} ms]", table.name, shard, (e - s) / 1000000.0);
     }
   }
 
@@ -123,7 +140,7 @@ public class DefaultBlurIndexWarmup extends BlurIndexWarmup {
     for (String field : preCacheCols) {
       maybePause(pauseWarmup);
       try {
-        indexWarmup.warm(reader, sampleIndex, field, context);
+        indexWarmup.warmFile(reader, sampleIndex, field, context);
       } catch (IOException e) {
         LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context, field);
       }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 12b756a..7d6cede 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -477,7 +477,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer {
     }
 
     BlurIndex index = tableContext.newInstanceBlurIndex(shardContext, dir, _mergeScheduler, _gc, _searchExecutor,
-        _indexCloser, _refresher);
+        _indexCloser, _refresher, _warmup);
 
     if (_clusterStatus.isReadOnly(true, _cluster, table)) {
       index = new BlurIndexReadOnly(index);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index 27960cd..e0c6b38 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -67,6 +67,7 @@ public class LocalIndexServer extends AbstractIndexServer {
   private final TableContext _tableContext;
   private final Closer _closer;
   private final boolean _ramDir;
+  private final BlurIndexWarmup _indexWarmup;
 
   public LocalIndexServer(TableDescriptor tableDescriptor) throws IOException {
     this(tableDescriptor, false);
@@ -81,6 +82,7 @@ public class LocalIndexServer extends AbstractIndexServer {
     _closer.register(new CloseableExecutorService(_searchExecutor));
     _ramDir = ramDir;
     getIndexes(_tableContext.getTable());
+    _indexWarmup = BlurIndexWarmup.getIndexWarmup(_tableContext.getBlurConfiguration());
   }
 
   @Override
@@ -156,7 +158,8 @@ public class LocalIndexServer extends AbstractIndexServer {
 
   private BlurIndex openIndex(String table, String shard, Directory dir) throws CorruptIndexException, IOException {
     ShardContext shardContext = ShardContext.create(_tableContext, shard);
-    BlurNRTIndex index = new BlurNRTIndex(shardContext, dir, _mergeScheduler, _gc, _searchExecutor, null, null);
+    BlurNRTIndex index = new BlurNRTIndex(shardContext, dir, _mergeScheduler, _gc, _searchExecutor, null, null,
+        _indexWarmup);
     return index;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
index 49fe965..f70401d 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
@@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.thrift.generated.Row;
@@ -37,13 +38,12 @@ import org.apache.lucene.store.Directory;
 public abstract class BlurIndex {
 
   private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1);
-
   private long _lastMemoryCheck = 0;
   private long _memoryUsage = 0;
 
   public BlurIndex(ShardContext shardContext, Directory directory, SharedMergeScheduler mergeScheduler,
       DirectoryReferenceFileGC gc, ExecutorService searchExecutor, BlurIndexCloser indexCloser,
-      BlurIndexRefresher refresher) throws IOException {
+      BlurIndexRefresher refresher, BlurIndexWarmup indexWarmup) throws IOException {
 
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
index e8a3c32..3009a87 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
@@ -28,7 +28,7 @@ public class BlurIndexReadOnly extends BlurIndex {
   private final BlurIndex _blurIndex;
 
   public BlurIndexReadOnly(BlurIndex blurIndex) throws IOException {
-    super(null, null, null, null, null, null, null);
+    super(null, null, null, null, null, null, null, null);
     _blurIndex = blurIndex;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
index db5301a..4071777 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
@@ -30,6 +30,7 @@ import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.codec.Blur022Codec;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.warmup.TraceableDirectory;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
@@ -52,11 +53,11 @@ public class BlurIndexReader extends BlurIndex {
   private BlurIndexRefresher _refresher;
   private final TableContext _tableContext;
   private final ShardContext _shardContext;
-  
+
   public BlurIndexReader(ShardContext shardContext, Directory directory, SharedMergeScheduler mergeScheduler,
       DirectoryReferenceFileGC gc, final ExecutorService searchExecutor, BlurIndexCloser indexCloser,
-      BlurIndexRefresher refresher) throws IOException {
-    super(shardContext, directory, mergeScheduler, gc, searchExecutor, indexCloser, refresher);
+      BlurIndexRefresher refresher, BlurIndexWarmup indexWarmup) throws IOException {
+    super(shardContext, directory, mergeScheduler, gc, searchExecutor, indexCloser, refresher, indexWarmup);
     _tableContext = shardContext.getTableContext();
     // This directory allows for warm up by adding tracing ability.
     _directory = new TraceableDirectory(directory);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReaderWarmer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReaderWarmer.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReaderWarmer.java
new file mode 100644
index 0000000..aa3b8d2
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReaderWarmer.java
@@ -0,0 +1,60 @@
+package org.apache.blur.manager.writer;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
+import org.apache.blur.manager.indexserver.DistributedIndexServer.ReleaseReader;
+import org.apache.blur.server.ShardContext;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
+
+public class BlurIndexReaderWarmer extends IndexReaderWarmer {
+
+  private static final Log LOG = LogFactory.getLog(BlurIndexReaderWarmer.class);
+
+  private final AtomicBoolean _isClosed;
+  private final AtomicLong _pause = new AtomicLong();
+  private final BlurIndexWarmup _indexWarmup;
+  private final TableDescriptor _table;
+  private final String _shard;
+
+  public BlurIndexReaderWarmer(ShardContext shardContext, AtomicBoolean isClosed, BlurIndexWarmup indexWarmup) {
+    _isClosed = isClosed;
+    _table = shardContext.getTableContext().getDescriptor();
+    _shard = shardContext.getShard();
+    _indexWarmup = indexWarmup;
+  }
+
+  @Override
+  public void warm(AtomicReader reader) throws IOException {
+    LOG.info("Warming reader [{0}]", reader);
+    ReleaseReader releaseReader = new ReleaseReader() {
+      @Override
+      public void release() throws IOException {
+
+      }
+    };
+    _indexWarmup.warmBlurIndex(_table, _shard, reader, _isClosed, releaseReader, _pause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
index e3b1058..f27ba44 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
@@ -36,6 +36,7 @@ import org.apache.blur.lucene.codec.Blur022Codec;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.warmup.TraceableDirectory;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
@@ -73,8 +74,8 @@ public class BlurIndexSimpleWriter extends BlurIndex {
 
   public BlurIndexSimpleWriter(ShardContext shardContext, Directory directory, SharedMergeScheduler mergeScheduler,
       DirectoryReferenceFileGC gc, final ExecutorService searchExecutor, BlurIndexCloser indexCloser,
-      BlurIndexRefresher refresher) throws IOException {
-    super(shardContext, directory, mergeScheduler, gc, searchExecutor, indexCloser, refresher);
+      BlurIndexRefresher refresher, BlurIndexWarmup indexWarmup) throws IOException {
+    super(shardContext, directory, mergeScheduler, gc, searchExecutor, indexCloser, refresher, indexWarmup);
     _searchThreadPool = searchExecutor;
     _shardContext = shardContext;
     _tableContext = _shardContext.getTableContext();
@@ -84,8 +85,7 @@ public class BlurIndexSimpleWriter extends BlurIndex {
     _conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
     _conf.setCodec(new Blur022Codec(_tableContext.getBlurConfiguration()));
     _conf.setSimilarity(_tableContext.getSimilarity());
-    AtomicBoolean stop = new AtomicBoolean();
-    _conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, stop, _isClosed));
+    _conf.setMergedSegmentWarmer(new BlurIndexReaderWarmer(shardContext, _isClosed, indexWarmup));
     TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);
     _conf.setMergeScheduler(mergeScheduler.getMergeScheduler());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index ad50599..e1192b1 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -42,6 +42,7 @@ import org.apache.blur.lucene.codec.Blur022Codec;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.warmup.TraceableDirectory;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.IndexSearcherClosableNRT;
 import org.apache.blur.server.ShardContext;
@@ -97,8 +98,8 @@ public class BlurNRTIndex extends BlurIndex {
 
   public BlurNRTIndex(ShardContext shardContext, Directory directory, SharedMergeScheduler mergeScheduler,
       DirectoryReferenceFileGC gc, final ExecutorService searchExecutor, BlurIndexCloser indexCloser,
-      BlurIndexRefresher refresher) throws IOException {
-    super(shardContext, directory, mergeScheduler, gc, searchExecutor, indexCloser, refresher);
+      BlurIndexRefresher refresher, BlurIndexWarmup indexWarmup) throws IOException {
+    super(shardContext, directory, mergeScheduler, gc, searchExecutor, indexCloser, refresher, indexWarmup);
     _tableContext = shardContext.getTableContext();
     _directory = directory;
     _shardContext = shardContext;
@@ -109,8 +110,7 @@ public class BlurNRTIndex extends BlurIndex {
     conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
     conf.setCodec(new Blur022Codec(_tableContext.getBlurConfiguration()));
     conf.setSimilarity(_tableContext.getSimilarity());
-    AtomicBoolean stop = new AtomicBoolean();
-    conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, stop, _isClosed));
+    conf.setMergedSegmentWarmer(new BlurIndexReaderWarmer(shardContext, _isClosed, indexWarmup));
 
     SnapshotDeletionPolicy sdp;
     if (snapshotsDirectoryExists()) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
deleted file mode 100644
index f77d4ef..0000000
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package org.apache.blur.manager.writer;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.blur.lucene.warmup.IndexTracerResult;
-import org.apache.blur.lucene.warmup.IndexWarmup;
-import org.apache.blur.server.ShardContext;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
-
-public class FieldBasedWarmer extends IndexReaderWarmer {
-
-  private static final Log LOG = LogFactory.getLog(FieldBasedWarmer.class);
-
-  private final ShardContext _shardContext;
-  private final AtomicBoolean _isClosed;
-  private final AtomicBoolean _stop;
-
-  public FieldBasedWarmer(ShardContext shardContext, AtomicBoolean isClosed, AtomicBoolean stop) {
-    _isClosed = isClosed;
-    _stop = stop;
-    _shardContext = shardContext;
-  }
-
-  @Override
-  public void warm(AtomicReader reader) throws IOException {
-    List<String> preCacheCols = _shardContext.getTableContext().getDescriptor().getPreCacheCols();
-    int maxSampleSize = 1000;
-    IndexWarmup indexWarmup = new IndexWarmup(_isClosed, _stop, maxSampleSize);
-    String context = _shardContext.getTableContext().getTable() + "/" + _shardContext.getShard();
-    Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(reader, context);
-    if (preCacheCols != null) {
-      warm(reader, preCacheCols, indexWarmup, sampleIndex, context, _isClosed);
-    } else {
-      Fields fields = reader.fields();
-      warm(reader, fields, indexWarmup, sampleIndex, context, _isClosed);
-    }
-  }
-
-  private void warm(AtomicReader reader, Iterable<String> preCacheCols, IndexWarmup indexWarmup,
-      Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean isClosed) {
-    for (String field : preCacheCols) {
-      try {
-        indexWarmup.warm(reader, sampleIndex, field, context);
-      } catch (IOException e) {
-        LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context, field);
-        LOG.error("Current sampleIndex [{0}]", sampleIndex);
-      }
-      if (isClosed.get()) {
-        LOG.info("Context [{0}] index closed", context);
-        return;
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 7c867ca..769d642 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -46,6 +46,7 @@ import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.manager.ReadInterceptor;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.manager.writer.BlurIndexCloser;
 import org.apache.blur.manager.writer.BlurIndexRefresher;
@@ -327,7 +328,7 @@ public class TableContext {
   @SuppressWarnings("unchecked")
   public BlurIndex newInstanceBlurIndex(ShardContext shardContext, Directory dir, SharedMergeScheduler mergeScheduler,
       DirectoryReferenceFileGC gc, ExecutorService searchExecutor, BlurIndexCloser indexCloser,
-      BlurIndexRefresher refresher) throws IOException {
+      BlurIndexRefresher refresher, BlurIndexWarmup indexWarmup) throws IOException {
 
     String className = _blurConfiguration.get(BLUR_SHARD_BLURINDEX_CLASS, BlurNRTIndex.class.getName());
 
@@ -339,7 +340,8 @@ public class TableContext {
     }
     Constructor<? extends BlurIndex> constructor = findConstructor(clazz);
     try {
-      return constructor.newInstance(shardContext, dir, mergeScheduler, gc, searchExecutor, indexCloser, refresher);
+      return constructor.newInstance(shardContext, dir, mergeScheduler, gc, searchExecutor, indexCloser, refresher,
+          indexWarmup);
     } catch (InstantiationException e) {
       throw new IOException(e);
     } catch (IllegalAccessException e) {
@@ -354,7 +356,8 @@ public class TableContext {
   private Constructor<? extends BlurIndex> findConstructor(Class<? extends BlurIndex> clazz) throws IOException {
     try {
       return clazz.getConstructor(new Class[] { ShardContext.class, Directory.class, SharedMergeScheduler.class,
-          DirectoryReferenceFileGC.class, ExecutorService.class, BlurIndexCloser.class, BlurIndexRefresher.class });
+          DirectoryReferenceFileGC.class, ExecutorService.class, BlurIndexCloser.class, BlurIndexRefresher.class,
+          BlurIndexWarmup.class });
     } catch (NoSuchMethodException e) {
       throw new IOException(e);
     } catch (SecurityException e) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 8339c42..7293cd4 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -33,8 +33,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCK_CACHE_VERSION
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FETCHCOUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
-import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
-import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_THROTTLE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_MERGE_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
@@ -69,7 +67,6 @@ import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus;
 import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.manager.indexserver.BlurServerShutDown;
 import org.apache.blur.manager.indexserver.BlurServerShutDown.BlurShutdown;
-import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup;
 import org.apache.blur.manager.indexserver.DistributedIndexServer;
 import org.apache.blur.manager.indexserver.DistributedLayoutFactory;
 import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl;
@@ -197,7 +194,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     final BlurIndexRefresher refresher = new BlurIndexRefresher();
 
     BlurFilterCache filterCache = getFilterCache(configuration);
-    BlurIndexWarmup indexWarmup = getIndexWarmup(configuration);
+    BlurIndexWarmup indexWarmup = BlurIndexWarmup.getIndexWarmup(configuration);
 
     DistributedLayoutFactory distributedLayoutFactory = DistributedLayoutFactoryImpl.getDistributedLayoutFactory(
         configuration, cluster, zooKeeper);
@@ -295,25 +292,4 @@ public class ThriftBlurShardServer extends ThriftServer {
     return new DefaultBlurFilterCache(configuration);
   }
 
-  private static BlurIndexWarmup getIndexWarmup(BlurConfiguration configuration) {
-    String blurFilterCacheClass = configuration.get(BLUR_SHARD_INDEX_WARMUP_CLASS);
-    if (blurFilterCacheClass != null && blurFilterCacheClass.isEmpty()) {
-      if (!blurFilterCacheClass.equals("org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup")) {
-        try {
-          Class<?> clazz = Class.forName(blurFilterCacheClass);
-          return (BlurIndexWarmup) clazz.newInstance();
-        } catch (Exception e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-    long totalThrottle = configuration.getLong(BLUR_SHARD_INDEX_WARMUP_THROTTLE, 30000000);
-    int totalThreadCount = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 30000000);
-    long warmupBandwidthThrottleBytesPerSec = totalThrottle / totalThreadCount;
-    if (warmupBandwidthThrottleBytesPerSec <= 0) {
-      LOG.warn("Invalid values of either [{0} = {1}] or [{2} = {3}], needs to be greater then 0",
-          BLUR_SHARD_INDEX_WARMUP_THROTTLE, totalThrottle, BLUR_SHARD_WARMUP_THREAD_COUNT, totalThreadCount);
-    }
-    return new DefaultBlurIndexWarmup(warmupBandwidthThrottleBytesPerSec);
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/test/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmupTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmupTest.java b/blur-core/src/test/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmupTest.java
new file mode 100644
index 0000000..4f69e60
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmupTest.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.manager.indexserver;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.blur.lucene.warmup.TraceableDirectory;
+import org.apache.blur.manager.indexserver.DistributedIndexServer.ReleaseReader;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class DefaultBlurIndexWarmupTest {
+
+  private Random _random = new Random();
+  private int _numberOfFields = 10;
+
+  @Test
+  public void testDefaultBlurIndexWarmupTestFullIndexReader() throws IOException {
+    File file = new File("./target/tmp/DefaultBlurIndexWarmupTest-test");
+    Directory dir = FSDirectory.open(file);
+
+    Directory directory = new TraceableDirectory(dir);
+    IndexReader indexReader = getIndexReader(directory);
+
+    DefaultBlurIndexWarmup indexWarmup = new DefaultBlurIndexWarmup(10000000);
+    AtomicBoolean isClosed = new AtomicBoolean();
+
+    AtomicLong pauseWarmup = new AtomicLong();
+    ReleaseReader releaseReader = new ReleaseReader() {
+      @Override
+      public void release() throws IOException {
+
+      }
+    };
+    String shard = "shard";
+    TableDescriptor table = new TableDescriptor();
+    table.setName("test");
+
+    long t1 = System.nanoTime();
+    indexWarmup.warmBlurIndex(table, shard, indexReader, isClosed, releaseReader, pauseWarmup);
+    long t2 = System.nanoTime();
+    System.out.println((t2 - t1) / 1000000.0);
+  }
+
+  @Test
+  public void testDefaultBlurIndexWarmupTestOneSegment() throws IOException {
+    File file = new File("./target/tmp/DefaultBlurIndexWarmupTest-test");
+    Directory dir = FSDirectory.open(file);
+
+    Directory directory = new TraceableDirectory(dir);
+    IndexReader indexReader = getIndexReader(directory);
+
+    DefaultBlurIndexWarmup indexWarmup = new DefaultBlurIndexWarmup(10000000);
+    AtomicBoolean isClosed = new AtomicBoolean();
+
+    AtomicLong pauseWarmup = new AtomicLong();
+    ReleaseReader releaseReader = new ReleaseReader() {
+      @Override
+      public void release() throws IOException {
+
+      }
+    };
+    String shard = "shard";
+    TableDescriptor table = new TableDescriptor();
+    table.setName("test");
+
+    AtomicReader reader = indexReader.leaves().iterator().next().reader();
+
+    long t1 = System.nanoTime();
+    indexWarmup.warmBlurIndex(table, shard, reader, isClosed, releaseReader, pauseWarmup);
+    long t2 = System.nanoTime();
+    System.out.println((t2 - t1) / 1000000.0);
+  }
+
+  private IndexReader getIndexReader(Directory directory) throws IOException {
+    if (!DirectoryReader.indexExists(directory)) {
+      long t1 = System.nanoTime();
+      populate(directory);
+      long t2 = System.nanoTime();
+      System.out.println((t2 - t1) / 1000000.0);
+    }
+    return DirectoryReader.open(directory);
+  }
+
+  private void populate(Directory directory) throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
+    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
+    mergePolicy.setUseCompoundFile(false);
+    IndexWriter writer = new IndexWriter(directory, conf);
+    addDocs(writer);
+    writer.close();
+  }
+
+  private void addDocs(IndexWriter writer) throws IOException {
+    for (int i = 0; i < 2000; i++) {
+      writer.addDocument(getDoc());
+    }
+  }
+
+  private Iterable<? extends IndexableField> getDoc() {
+    Document document = new Document();
+    document.add(new TextField(getFieldName(), getText(), Store.YES));
+    return document;
+  }
+
+  private String getText() {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < 1000; i++) {
+      builder.append(getWord()).append(' ');
+    }
+    return builder.toString();
+  }
+
+  private String getWord() {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < 10; i++) {
+      builder.append(getChar());
+    }
+    return builder.toString();
+  }
+
+  private char getChar() {
+    return (char) (_random.nextInt(26) + 'a');
+  }
+
+  private String getFieldName() {
+    return "test" + _random.nextInt(_numberOfFields);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexNRTSimpleTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexNRTSimpleTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexNRTSimpleTest.java
index faf83eb..96178c4 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexNRTSimpleTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexNRTSimpleTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService;
 import org.apache.blur.BlurConfiguration;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
+import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
@@ -64,6 +65,7 @@ public class BlurIndexNRTSimpleTest {
   private String uuid;
   private BlurIndexRefresher _refresher;
   private BlurIndexCloser _closer;
+  private DefaultBlurIndexWarmup indexWarmup;
 
   @Before
   public void setup() throws IOException {
@@ -79,21 +81,22 @@ public class BlurIndexNRTSimpleTest {
     service = Executors.newThreadPool("test", 10);
     _refresher = new BlurIndexRefresher();
     _closer = new BlurIndexCloser();
+    indexWarmup = new DefaultBlurIndexWarmup(1000000);
   }
 
   private void setupWriter(Configuration configuration, long refresh, boolean reload) throws IOException {
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setName("test-table");
     /*
-     * if reload is set to true...we create a new writer instance pointing
-     * to the same location as the old one.....
-     * so previous writer instances should be closed
+     * if reload is set to true...we create a new writer instance pointing to
+     * the same location as the old one..... so previous writer instances should
+     * be closed
      */
-    
+
     if (!reload && uuid == null) {
       uuid = UUID.randomUUID().toString();
     }
-    
+
     tableDescriptor.setTableUri(new File(base, "table-store-" + uuid).toURI().toString());
     tableDescriptor.putToTableProperties("blur.shard.time.between.refreshs", Long.toString(refresh));
 
@@ -102,7 +105,8 @@ public class BlurIndexNRTSimpleTest {
     path.mkdirs();
     FSDirectory directory = FSDirectory.open(path);
     ShardContext shardContext = ShardContext.create(tableContext, "test-shard-" + uuid);
-    writer = new BlurIndexSimpleWriter(shardContext, directory, mergeScheduler, gc, service, _closer, _refresher);
+    writer = new BlurIndexSimpleWriter(shardContext, directory, mergeScheduler, gc, service, _closer, _refresher,
+        indexWarmup);
   }
 
   @After
@@ -136,8 +140,9 @@ public class BlurIndexNRTSimpleTest {
     Trace.setStorage(new BaseTraceStorage(new BlurConfiguration()) {
       @Override
       public void close() throws IOException {
-        
+
       }
+
       @Override
       public void store(TraceCollector collector) {
         System.out.println(collector.toJson());
@@ -208,54 +213,59 @@ public class BlurIndexNRTSimpleTest {
     return row;
   }
 
-//  @Test
-//  public void testCreateSnapshot() throws IOException {
-//    setupWriter(configuration, 5, false);
-//    writer.createSnapshot("test_snapshot");
-//    assertTrue(writer.getSnapshots().contains("test_snapshot"));
-//    
-//    // check that the file is persisted
-//    Path snapshotsDirPath = writer.getSnapshotsDirectoryPath();
-//    FileSystem fileSystem = snapshotsDirPath.getFileSystem(new Configuration());
-//    Path snapshotFilePath = new Path(snapshotsDirPath, "test_snapshot");
-//    assertTrue(fileSystem.exists(snapshotFilePath));
-//    
-//    // create a new writer instance and test whether the snapshots are loaded properly
-//    writer.close();
-//    setupWriter(configuration, 5, true);
-//    assertTrue(writer.getSnapshots().contains("test_snapshot"));
-//  }
-//  
-//  
-//  @Test
-//  public void testRemoveSnapshots() throws IOException {
-//    setupWriter(configuration, 5, false);
-//    Path snapshotsDirPath = writer.getSnapshotsDirectoryPath();
-//    FileSystem fileSystem = snapshotsDirPath.getFileSystem(new Configuration());
-//    fileSystem.mkdirs(snapshotsDirPath);
-//    
-//    // create 2 files in snapshots sub-dir
-//    Path snapshotFile1 = new Path(snapshotsDirPath, "test_snapshot1");
-//    Path snapshotFile2 = new Path(snapshotsDirPath, "test_snapshot2");
-//    
-//    BufferedWriter br1 = new BufferedWriter(new OutputStreamWriter(fileSystem.create(snapshotFile1, true)));
-//    br1.write("segments_1");
-//    br1.close();
-//    
-//    BufferedWriter br2 = new BufferedWriter(new OutputStreamWriter(fileSystem.create(snapshotFile2, true)));
-//    br2.write("segments_1");
-//    br2.close();
-//    
-//    // re-load the writer to load the snpshots
-//    writer.close();
-//    setupWriter(configuration, 5, true);
-//    assertEquals(writer.getSnapshots().size(), 2);
-//    
-//    
-//    writer.removeSnapshot("test_snapshot2");
-//    assertEquals(writer.getSnapshots().size(), 1);
-//    assertTrue(!writer.getSnapshots().contains("test_snapshot2"));
-//    assertTrue(!fileSystem.exists(snapshotFile2));
-//
-//  }
+  // @Test
+  // public void testCreateSnapshot() throws IOException {
+  // setupWriter(configuration, 5, false);
+  // writer.createSnapshot("test_snapshot");
+  // assertTrue(writer.getSnapshots().contains("test_snapshot"));
+  //
+  // // check that the file is persisted
+  // Path snapshotsDirPath = writer.getSnapshotsDirectoryPath();
+  // FileSystem fileSystem = snapshotsDirPath.getFileSystem(new
+  // Configuration());
+  // Path snapshotFilePath = new Path(snapshotsDirPath, "test_snapshot");
+  // assertTrue(fileSystem.exists(snapshotFilePath));
+  //
+  // // create a new writer instance and test whether the snapshots are loaded
+  // properly
+  // writer.close();
+  // setupWriter(configuration, 5, true);
+  // assertTrue(writer.getSnapshots().contains("test_snapshot"));
+  // }
+  //
+  //
+  // @Test
+  // public void testRemoveSnapshots() throws IOException {
+  // setupWriter(configuration, 5, false);
+  // Path snapshotsDirPath = writer.getSnapshotsDirectoryPath();
+  // FileSystem fileSystem = snapshotsDirPath.getFileSystem(new
+  // Configuration());
+  // fileSystem.mkdirs(snapshotsDirPath);
+  //
+  // // create 2 files in snapshots sub-dir
+  // Path snapshotFile1 = new Path(snapshotsDirPath, "test_snapshot1");
+  // Path snapshotFile2 = new Path(snapshotsDirPath, "test_snapshot2");
+  //
+  // BufferedWriter br1 = new BufferedWriter(new
+  // OutputStreamWriter(fileSystem.create(snapshotFile1, true)));
+  // br1.write("segments_1");
+  // br1.close();
+  //
+  // BufferedWriter br2 = new BufferedWriter(new
+  // OutputStreamWriter(fileSystem.create(snapshotFile2, true)));
+  // br2.write("segments_1");
+  // br2.close();
+  //
+  // // re-load the writer to load the snpshots
+  // writer.close();
+  // setupWriter(configuration, 5, true);
+  // assertEquals(writer.getSnapshots().size(), 2);
+  //
+  //
+  // writer.removeSnapshot("test_snapshot2");
+  // assertEquals(writer.getSnapshots().size(), 1);
+  // assertTrue(!writer.getSnapshots().contains("test_snapshot2"));
+  // assertTrue(!fileSystem.exists(snapshotFile2));
+  //
+  // }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
index cb7d649..91c89eb 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService;
 
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
+import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
@@ -60,6 +61,7 @@ public class BlurIndexReaderTest {
   private BlurIndexRefresher refresher;
   private BlurIndexCloser indexCloser;
   private FSDirectory directory;
+  private DefaultBlurIndexWarmup indexWarmup;
 
   @Before
   public void setup() throws IOException {
@@ -73,6 +75,7 @@ public class BlurIndexReaderTest {
 
     configuration = new Configuration();
     service = Executors.newThreadPool("test", 1);
+    indexWarmup = new DefaultBlurIndexWarmup(1000000);
 
   }
 
@@ -90,7 +93,7 @@ public class BlurIndexReaderTest {
     ShardContext shardContext = ShardContext.create(tableContext, "test-shard");
     refresher = new BlurIndexRefresher();
     indexCloser = new BlurIndexCloser();
-    reader = new BlurIndexReader(shardContext, directory, null, null, null, indexCloser, refresher);
+    reader = new BlurIndexReader(shardContext, directory, null, null, null, indexCloser, refresher, indexWarmup);
   }
 
   @After

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
index 9f2ffc3..4abb9b9 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
@@ -30,6 +30,8 @@ import java.util.concurrent.ExecutorService;
 
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
+import org.apache.blur.manager.indexserver.BlurIndexWarmup;
+import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
@@ -62,6 +64,7 @@ public class BlurNRTIndexTest {
   private DirectoryReferenceFileGC gc;
   private SharedMergeScheduler mergeScheduler;
   private String uuid;
+  private BlurIndexWarmup indexWarmup;
 
   @Before
   public void setup() throws IOException {
@@ -75,6 +78,7 @@ public class BlurNRTIndexTest {
 
     configuration = new Configuration();
     service = Executors.newThreadPool("test", 10);
+    indexWarmup = new DefaultBlurIndexWarmup(1000000);
   }
 
   private void setupWriter(Configuration configuration, long refresh, boolean reload) throws IOException {
@@ -98,7 +102,7 @@ public class BlurNRTIndexTest {
     path.mkdirs();
     FSDirectory directory = FSDirectory.open(path);
     ShardContext shardContext = ShardContext.create(tableContext, "test-shard-" + uuid);
-    writer = new BlurNRTIndex(shardContext, directory, mergeScheduler, gc, service, null, null);
+    writer = new BlurNRTIndex(shardContext, directory, mergeScheduler, gc, service, null, null, indexWarmup);
   }
 
   @After

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
index bc26283..33d9161 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
@@ -25,6 +25,7 @@ import java.util.Map.Entry;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.blur.index.ExitableReader.ExitableFilterAtomicReader;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.warmup.IndexTracerResult.FILE_TYPE;
@@ -39,6 +40,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.OpenBitSet;
 
 /**
  * IndexWarmup is used to pre-read portions of the index by field. Usage:<br/>
@@ -98,14 +100,64 @@ public class IndexWarmup {
     runTrace.set(Boolean.FALSE);
   }
 
-  public void warm(IndexReader reader, Map<String, List<IndexTracerResult>> sampleIndex, String fieldName,
+  public void warmFile(IndexReader reader, Map<String, List<IndexTracerResult>> sampleIndex, String fieldName,
       String context) throws IOException {
     for (Entry<String, List<IndexTracerResult>> segment : sampleIndex.entrySet()) {
-      warm(reader, segment.getValue(), fieldName, context);
+      warmFile(reader, segment.getValue(), fieldName, context);
     }
   }
 
-  public void warm(IndexReader reader, List<IndexTracerResult> traces, String fieldName, String context)
+  public void getFilePositionsToWarm(IndexReader reader, Map<String, List<IndexTracerResult>> sampleIndex,
+      String fieldName, String context, Map<String, OpenBitSet> filePartsToWarm, int blockSize) throws IOException {
+    for (Entry<String, List<IndexTracerResult>> segment : sampleIndex.entrySet()) {
+      getFilePositionsToWarm(reader, segment.getValue(), fieldName, context, filePartsToWarm, blockSize);
+    }
+  }
+
+  public void getFilePositionsToWarm(IndexReader reader, List<IndexTracerResult> traces, String fieldName,
+      String context, Map<String, OpenBitSet> filePartsToWarm, int blockSize) throws IOException {
+    int index = find(traces, fieldName);
+    if (index < 0) {
+      // not found
+      return;
+    }
+    IndexTracerResult trace = traces.get(index);
+    for (FILE_TYPE type : FILE_TYPE.values()) {
+      if (trace.isFilePositionCaptured(type)) {
+        long startingPosition = trace.getPosition(type);
+        long endingPosition = Long.MAX_VALUE;
+        int nextIndex = findNextSetTrace(traces, index + 1, type);
+        if (nextIndex >= 0) {
+          IndexTracerResult next = traces.get(nextIndex);
+          endingPosition = next.getPosition(type);
+        }
+        String fileName = trace.getFileName(type);
+        String segmentName = trace.getSegmentName();
+        getFilePositionsToWarm(reader, segmentName, fileName, fieldName, startingPosition, endingPosition, context,
+            filePartsToWarm, blockSize);
+      }
+    }
+  }
+
+  private void getFilePositionsToWarm(IndexReader reader, String segmentName, String fileName, String fieldName,
+      long startingPosition, long endingPosition, String context, Map<String, OpenBitSet> filePartsToWarm, int blockSize)
+      throws IOException {
+    Directory dir = getDirectory(reader, segmentName, context);
+    long fileLength = dir.fileLength(fileName);
+    if (endingPosition == Long.MAX_VALUE) {
+      endingPosition = fileLength - 1;
+    }
+    OpenBitSet openBitSet = filePartsToWarm.get(fileName);
+    if (openBitSet == null) {
+      openBitSet = new OpenBitSet((long) Math.ceil(fileLength / (double) blockSize));
+      filePartsToWarm.put(fileName, openBitSet);
+    }
+    long startingBlock = startingPosition / blockSize;
+    long endingBlock = endingPosition / blockSize;
+    openBitSet.set(startingBlock, endingBlock);
+  }
+
+  void warmFile(IndexReader reader, List<IndexTracerResult> traces, String fieldName, String context)
       throws IOException {
     int index = find(traces, fieldName);
     if (index < 0) {
@@ -124,13 +176,13 @@ public class IndexWarmup {
         }
         String fileName = trace.getFileName(type);
         String segmentName = trace.getSegmentName();
-        warm(reader, segmentName, fileName, fieldName, startingPosition, endingPosition, context);
+        warmFile(reader, segmentName, fileName, fieldName, startingPosition, endingPosition, context);
       }
     }
   }
 
-  private void warm(IndexReader reader, String segmentName, String fileName, String fieldName, long startingPosition,
-      long endingPosition, String context) throws IOException {
+  private void warmFile(IndexReader reader, String segmentName, String fileName, String fieldName,
+      long startingPosition, long endingPosition, String context) throws IOException {
     Directory dir = getDirectory(reader, segmentName, context);
     if (dir == null) {
       LOG.info("Context [{0}] cannot find segment [{1}]", context, segmentName);
@@ -167,7 +219,7 @@ public class IndexWarmup {
           }
         }
         int len = (int) Math.min(length, buf.length);
-        input.readBytes(buf, 0, len);
+        input.readBytes(buf, 0, len, false);
         length -= len;
         bytesReadPerPass += len;
       }
@@ -177,7 +229,7 @@ public class IndexWarmup {
         seconds = 1;
       }
       double rateMbPerSec = (bytesReadPerPass / seconds) / 1000 / 1000;
-      LOG.debug("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at rate of [{4} MB/s]", fieldName,
+      LOG.info("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at rate of [{4} MB/s]", fieldName,
           fileName, 100, context, rateMbPerSec);
     } finally {
       input.close();
@@ -205,6 +257,34 @@ public class IndexWarmup {
     return null;
   }
 
+  private Directory getDirectory(IndexReader reader, String context) {
+    if (reader instanceof AtomicReader) {
+      return getDirectory((AtomicReader) reader, context);
+    }
+    for (IndexReaderContext ctext : reader.getContext().leaves()) {
+      if (_isClosed.get() || _stop.get()) {
+        LOG.info("Context [{0}] index closed", context);
+        return null;
+      }
+      AtomicReaderContext atomicReaderContext = (AtomicReaderContext) ctext;
+      AtomicReader atomicReader = atomicReaderContext.reader();
+      if (atomicReader instanceof SegmentReader) {
+        SegmentReader segmentReader = (SegmentReader) atomicReader;
+        return segmentReader.directory();
+      }
+    }
+    return null;
+  }
+
+  private Directory getDirectory(AtomicReader reader, String context) {
+    if (reader instanceof ExitableFilterAtomicReader) {
+      return getDirectory(((ExitableFilterAtomicReader) reader).getOriginalReader(), context);
+    } else if (reader instanceof SegmentReader) {
+      return ((SegmentReader) reader).directory();
+    }
+    return null;
+  }
+
   private Directory getDirectory(AtomicReader atomicReader, String segmentName, String context) {
     if (atomicReader instanceof SegmentReader) {
       SegmentReader segmentReader = (SegmentReader) atomicReader;
@@ -313,4 +393,106 @@ public class IndexWarmup {
     }
   }
 
+  public void warmFile(IndexReader indexReader, Map<String, OpenBitSet> filePartsToWarm, String context, int blockSize,
+      int bufferSize) throws IOException {
+    Directory directory = getDirectory(indexReader, context);
+    ThrottledIndexInput input = null;
+    long s = System.nanoTime();
+    double _targetThrouhput = _maxBytesPerSec / 1000.0 / 1000.0;
+    for (Entry<String, OpenBitSet> e : filePartsToWarm.entrySet()) {
+      input = warmFile(directory, e.getKey(), e.getValue(), blockSize, context, bufferSize, input);
+      if (input != null) {
+        long end = System.nanoTime();
+        double seconds = (end - s) / 1000000000.0;
+        double rateMbPerSec = (input.getTotalBytesRead() / seconds) / 1000 / 1000;
+        LOG.debug(
+            "Context [{0}] warming file [{1}] is [{2}%] complete " + "at rate of [{3} MB/s] target was [{4} MB/s]",
+            context, e.getKey(), 100, rateMbPerSec, _targetThrouhput);
+      }
+    }
+    if (input != null) {
+      long end = System.nanoTime();
+      double seconds = (end - s) / 1000000000.0;
+      double rateMbPerSec = (input.getTotalBytesRead() / seconds) / 1000 / 1000;
+      LOG.info("Finished warming context [{2}] [{0} MB/s] target was [{1} MB/s]", rateMbPerSec, _targetThrouhput,
+          context);
+    }
+  }
+
+  private ThrottledIndexInput warmFile(Directory dir, String fileName, OpenBitSet bitSet, int blockSize,
+      String context, int bufferSize, ThrottledIndexInput lastInput) throws IOException {
+    if (bufferSize < blockSize) {
+      throw new IOException("Buffer Size [" + bufferSize + "] cannot be less than Block Size [" + blockSize + "]");
+    }
+    if (_isClosed.get() || _stop.get()) {
+      LOG.info("Context [{0}] index closed", context);
+      return null;
+    }
+    ThrottledIndexInput input;
+    if (lastInput == null) {
+      input = new ThrottledIndexInput(dir.openInput(fileName, IOContext.READ), _maxBytesPerSec);
+    } else {
+      input = new ThrottledIndexInput(dir.openInput(fileName, IOContext.READ), _maxBytesPerSec,
+          lastInput.getTotalBytesRead(), lastInput.getTotalSleepTime(), lastInput.getStartTime());
+    }
+    try {
+      long length = input.length();
+      final long totalAmountToBeRead = bitSet.capacity() * blockSize;
+      byte[] buf = new byte[bufferSize];
+      int maxNumberOfContiguousBlocks = bufferSize / blockSize;
+      long start = System.nanoTime();
+      final long originalStart = start;
+      long bytesReadPerPass = 0;
+      long totalBytesRead = 0;
+      long blockPosition = 0;
+      while ((blockPosition = bitSet.nextSetBit(blockPosition)) != -1) {
+        long now = System.nanoTime();
+        if (start + _5_SECONDS < now) {
+          double seconds = (now - start) / 1000000000.0;
+          double rateMbPerSec = (bytesReadPerPass / seconds) / 1000 / 1000;
+          double complete = (((double) totalAmountToBeRead - (double) totalBytesRead) / (double) totalAmountToBeRead) * 100.0;
+          LOG.debug("Context [{0}] warming file [{1}] is [{2}%] complete at rate of [{3} MB/s]", context, fileName,
+              complete, rateMbPerSec);
+          start = System.nanoTime();
+          bytesReadPerPass = 0;
+          if (_isClosed.get()) {
+            LOG.info("Context [{0}] index closed", context);
+            return null;
+          }
+        }
+        long currentPosition = input.getFilePointer();
+        long newPosition = blockPosition * blockSize;
+        if (newPosition != currentPosition) {
+          input.seek(newPosition);
+        }
+        int numberOfContiguousBlocks = getNumberOfContiguousBlock(maxNumberOfContiguousBlocks, bitSet, blockPosition);
+        int len = (int) Math.min(blockSize * numberOfContiguousBlocks, length - newPosition);
+        input.readBytes(buf, 0, len);
+        bytesReadPerPass += len;
+        totalBytesRead += len;
+        blockPosition += numberOfContiguousBlocks;
+        if (_isClosed.get()) {
+          LOG.info("Context [{0}] index closed", context);
+          return null;
+        }
+      }
+      long now = System.nanoTime();
+      double seconds = (now - originalStart) / 1000000000.0;
+      double rateMbPerSec = (totalBytesRead / seconds) / 1000 / 1000;
+      LOG.debug("Context [{0}] warming file [{1}] is [{2}%] complete at rate of [{3} MB/s]", context, fileName, 100,
+          rateMbPerSec);
+      return input;
+    } finally {
+      input.close();
+    }
+  }
+
+  private int getNumberOfContiguousBlock(int maxNumberOfContiguousBlocks, OpenBitSet bitSet, long blockPosition) {
+    for (int i = 0; i < maxNumberOfContiguousBlocks; i++) {
+      if (!bitSet.get(i + blockPosition)) {
+        return i;
+      }
+    }
+    return maxNumberOfContiguousBlocks;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
index 4a7f862..0b6d612 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
@@ -34,11 +34,11 @@ import org.apache.lucene.store.IndexInput;
  */
 public class ThrottledIndexInput extends IndexInput {
 
-  private static final long SLEEP_DURATION_MS = 50;
+  private static final long SLEEP_DURATION_MS = 1;
 
   private final IndexInput _rawStream;
   private final double _maxBytesPerSec;
-  private final long _startTime = System.nanoTime();
+  private final long _startTime;
 
   private long _bytesRead = 0;
   private long _totalSleepTime = 0;
@@ -47,6 +47,17 @@ public class ThrottledIndexInput extends IndexInput {
     super("ThrottledIndexInput(" + rawStream.toString() + ")");
     _rawStream = rawStream;
     _maxBytesPerSec = maxBytesPerSec;
+    _startTime = System.nanoTime();
+  }
+
+  public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec, long bytesRead, long totalSleepTime,
+      long startTime) {
+    super("ThrottledIndexInput(" + rawStream.toString() + ")");
+    _rawStream = rawStream;
+    _maxBytesPerSec = maxBytesPerSec;
+    _bytesRead = bytesRead;
+    _totalSleepTime = totalSleepTime;
+    _startTime = startTime;
   }
 
   /** @inheritDoc */
@@ -146,4 +157,8 @@ public class ThrottledIndexInput extends IndexInput {
         + ", bytesPerSec=" + getBytesPerSec() + ", totalSleepTime=" + _totalSleepTime + '}';
   }
 
+  public long getStartTime() {
+    return _startTime;
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java b/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java
new file mode 100644
index 0000000..2fcc969
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.warmup;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class IndexWarmupTest {
+
+  private Random _random = new Random();
+  private int _numberOfFields = 10;
+
+  @Test
+  public void testIndexWarmup() throws IOException {
+    File file = new File("./target/tmp/indexwarmup-test");
+    Directory dir = FSDirectory.open(file);
+
+    Directory directory = new TraceableDirectory(new SlowAccessDirectory(dir));
+    IndexReader indexReader = getIndexReader(directory);
+    int maxSampleSize = 1000;
+    int blockSize = 8192;
+    long totalLookups = 0;
+    for (String s : directory.listAll()) {
+      if (s.endsWith(".pos") || s.endsWith(".doc") || s.endsWith(".tim")) {
+        long fileLength = directory.fileLength(s);
+        long maxHits = (long) Math.ceil(fileLength / (double) blockSize);
+        totalLookups += maxHits;
+        System.out.println("file [" + s + "] size [" + fileLength + "] maxhits [" + maxHits + "]");
+      }
+    }
+    AtomicBoolean stop = new AtomicBoolean();
+    AtomicBoolean isClosed = new AtomicBoolean();
+    SlowAccessDirectory._reads.set(0);
+    IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, Long.MAX_VALUE);
+    long t1 = System.nanoTime();
+    Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(indexReader, "test");
+    long sampleReads = SlowAccessDirectory._reads.get();
+    SlowAccessDirectory._reads.set(0);
+    long t2 = System.nanoTime();
+    for (int i = 0; i < _numberOfFields; i++) {
+      indexWarmup.warmFile(indexReader, sampleIndex, "test" + i, "test");
+    }
+    long t3 = System.nanoTime();
+    System.out.println((t2 - t1) / 1000000.0 + " " + (t3 - t2) / 1000000.0);
+    System.out.println(totalLookups + " " + sampleReads + " " + SlowAccessDirectory._reads.get());
+  }
+
+  @Test
+  public void testIndexWarmupBitSet() throws IOException {
+    File file = new File("./target/tmp/indexwarmup-test");
+    Directory dir = FSDirectory.open(file);
+
+    Directory directory = new TraceableDirectory(new SlowAccessDirectory(dir));
+    IndexReader indexReader = getIndexReader(directory);
+    int maxSampleSize = 1000;
+    int blockSize = 8192;
+    // int blockSize = 1024 * 1024;
+    long totalLookups = 0;
+    for (String s : directory.listAll()) {
+      if (s.endsWith(".pos") || s.endsWith(".doc") || s.endsWith(".tim")) {
+        long fileLength = directory.fileLength(s);
+        long maxHits = (long) Math.ceil(fileLength / (double) blockSize);
+        totalLookups += maxHits;
+        System.out.println("file [" + s + "] size [" + fileLength + "] maxhits [" + maxHits + "]");
+      }
+    }
+    AtomicBoolean stop = new AtomicBoolean();
+    AtomicBoolean isClosed = new AtomicBoolean();
+    SlowAccessDirectory._reads.set(0);
+    IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, Long.MAX_VALUE);
+    long t1 = System.nanoTime();
+    Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(indexReader, "test");
+    long sampleReads = SlowAccessDirectory._reads.get();
+    SlowAccessDirectory._reads.set(0);
+    long t2 = System.nanoTime();
+    Map<String, OpenBitSet> filePartsToWarm = new HashMap<String, OpenBitSet>();
+    for (int i = 0; i < _numberOfFields; i++) {
+      indexWarmup.getFilePositionsToWarm(indexReader, sampleIndex, "test" + i, "test", filePartsToWarm, blockSize);
+    }
+    indexWarmup.warmFile(indexReader, filePartsToWarm, "test", blockSize, 1024 * 1024);
+    long t3 = System.nanoTime();
+
+    for (Entry<String, OpenBitSet> e : filePartsToWarm.entrySet()) {
+      OpenBitSet bitSet = e.getValue();
+      System.out.println(bitSet.length() + " " + bitSet.cardinality());
+    }
+
+    System.out.println((t2 - t1) / 1000000.0 + " " + (t3 - t2) / 1000000.0);
+    System.out.println(totalLookups + " " + sampleReads + " " + SlowAccessDirectory._reads.get());
+  }
+
+  private IndexReader getIndexReader(Directory directory) throws IOException {
+    if (!DirectoryReader.indexExists(directory)) {
+      long t1 = System.nanoTime();
+      populate(directory);
+      long t2 = System.nanoTime();
+      System.out.println((t2 - t1) / 1000000.0);
+    }
+    return DirectoryReader.open(directory);
+  }
+
+  private void populate(Directory directory) throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
+    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
+    mergePolicy.setUseCompoundFile(false);
+    IndexWriter writer = new IndexWriter(directory, conf);
+    addDocs(writer);
+    writer.close();
+  }
+
+  private void addDocs(IndexWriter writer) throws IOException {
+    for (int i = 0; i < 20000; i++) {
+      writer.addDocument(getDoc());
+    }
+  }
+
+  private Iterable<? extends IndexableField> getDoc() {
+    Document document = new Document();
+    document.add(new TextField(getFieldName(), getText(), Store.YES));
+    return document;
+  }
+
+  private String getText() {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < 1000; i++) {
+      builder.append(getWord()).append(' ');
+    }
+    return builder.toString();
+  }
+
+  private String getWord() {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < 10; i++) {
+      builder.append(getChar());
+    }
+    return builder.toString();
+  }
+
+  private char getChar() {
+    return (char) (_random.nextInt(26) + 'a');
+  }
+
+  private String getFieldName() {
+    return "test" + _random.nextInt(_numberOfFields);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6db78b/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java b/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
new file mode 100644
index 0000000..f9aff08
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.warmup;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.store.BufferedIndexInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+
+public class SlowAccessDirectory extends Directory {
+  private final Directory _directory;
+
+  public static AtomicLong _reads = new AtomicLong();
+
+  static class SlowAccessIndexInput extends BufferedIndexInput {
+
+    private IndexInput _indexInput;
+
+    protected SlowAccessIndexInput(IndexInput indexInput) {
+      super(indexInput.toString(), 8192);
+      _indexInput = indexInput;
+    }
+
+    private void delay() throws IOException {
+      synchronized (this) {
+        try {
+          wait(1);
+        } catch (InterruptedException e) {
+          throw new IOException(e);
+        }
+      }
+    }
+
+    public void close() throws IOException {
+      _indexInput.close();
+    }
+
+    public long length() {
+      return _indexInput.length();
+    }
+
+    @Override
+    public SlowAccessIndexInput clone() {
+      SlowAccessIndexInput clone = (SlowAccessIndexInput) super.clone();
+      clone._indexInput = _indexInput.clone();
+      return clone;
+    }
+
+    @Override
+    protected void readInternal(byte[] b, int offset, int length) throws IOException {
+      delay();
+      _reads.incrementAndGet();
+      long filePointer = getFilePointer();
+      _indexInput.seek(filePointer);
+      _indexInput.readBytes(b, offset, length);
+    }
+
+    @Override
+    protected void seekInternal(long pos) throws IOException {
+
+    }
+
+  }
+
+  public SlowAccessDirectory(Directory directory) {
+    _directory = directory;
+  }
+
+  public String[] listAll() throws IOException {
+    return _directory.listAll();
+  }
+
+  public boolean fileExists(String name) throws IOException {
+    return _directory.fileExists(name);
+  }
+
+  public void deleteFile(String name) throws IOException {
+    _directory.deleteFile(name);
+  }
+
+  public long fileLength(String name) throws IOException {
+    return _directory.fileLength(name);
+  }
+
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
+    return _directory.createOutput(name, context);
+  }
+
+  public void sync(Collection<String> names) throws IOException {
+    _directory.sync(names);
+  }
+
+  public IndexInput openInput(String name, IOContext context) throws IOException {
+    return new SlowAccessIndexInput(_directory.openInput(name, context));
+  }
+
+  public Lock makeLock(String name) {
+    return _directory.makeLock(name);
+  }
+
+  public void clearLock(String name) throws IOException {
+    _directory.clearLock(name);
+  }
+
+  public void close() throws IOException {
+    _directory.close();
+  }
+
+  public void setLockFactory(LockFactory lockFactory) throws IOException {
+    _directory.setLockFactory(lockFactory);
+  }
+
+  public LockFactory getLockFactory() {
+    return _directory.getLockFactory();
+  }
+
+  public String getLockID() {
+    return _directory.getLockID();
+  }
+
+  public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
+    _directory.copy(to, src, dest, context);
+  }
+
+  public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException {
+    return _directory.createSlicer(name, context);
+  }
+}


Mime
View raw message