incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] Removing index warm up logic from Blur, it has been disabled for some time now.
Date Mon, 06 Oct 2014 13:10:29 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 13cef3ba4 -> e9dce5402


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
deleted file mode 100644
index 0b6d612..0000000
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.lucene.warmup;
-
-import java.io.IOException;
-
-import org.apache.lucene.store.IndexInput;
-
-/**
- * This class has been derived from the
- * org.apache.hadoop.tools.util.ThrottledInputStream class found in Hadoop.
- * 
- * The ThrottledIndexInput provides bandwidth throttling on a specified
- * IndexInput. It is implemented as a wrapper on top of another IndexInput
- * instance. The throttling works by examining the number of bytes read from the
- * underlying IndexInput from the beginning, and sleep()ing for a time interval
- * if the byte-transfer is found exceed the specified tolerable maximum. (Thus,
- * while the read-rate might exceed the maximum for a given short interval, the
- * average tends towards the specified maximum, overall.)
- */
-public class ThrottledIndexInput extends IndexInput {
-
-  private static final long SLEEP_DURATION_MS = 1;
-
-  private final IndexInput _rawStream;
-  private final double _maxBytesPerSec;
-  private final long _startTime;
-
-  private long _bytesRead = 0;
-  private long _totalSleepTime = 0;
-
-  public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec) {
-    super("ThrottledIndexInput(" + rawStream.toString() + ")");
-    _rawStream = rawStream;
-    _maxBytesPerSec = maxBytesPerSec;
-    _startTime = System.nanoTime();
-  }
-
-  public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec, long bytesRead, long
totalSleepTime,
-      long startTime) {
-    super("ThrottledIndexInput(" + rawStream.toString() + ")");
-    _rawStream = rawStream;
-    _maxBytesPerSec = maxBytesPerSec;
-    _bytesRead = bytesRead;
-    _totalSleepTime = totalSleepTime;
-    _startTime = startTime;
-  }
-
-  /** @inheritDoc */
-  @Override
-  public long getFilePointer() {
-    return _rawStream.getFilePointer();
-  }
-
-  /** @inheritDoc */
-  @Override
-  public void seek(long pos) throws IOException {
-    _rawStream.seek(pos);
-  }
-
-  /** @inheritDoc */
-  @Override
-  public long length() {
-    return _rawStream.length();
-  }
-
-  /** @inheritDoc */
-  @Override
-  public byte readByte() throws IOException {
-    throttle();
-    try {
-      return _rawStream.readByte();
-    } finally {
-      _bytesRead++;
-    }
-  }
-
-  /** @inheritDoc */
-  @Override
-  public void readBytes(byte[] b, int offset, int len) throws IOException {
-    throttle();
-    try {
-      _rawStream.readBytes(b, offset, len);
-    } finally {
-      _bytesRead += len;
-    }
-  }
-
-  /** @inheritDoc */
-  @Override
-  public void close() throws IOException {
-    _rawStream.close();
-  }
-
-  private void throttle() throws IOException {
-    while (getBytesPerSec() > _maxBytesPerSec) {
-      try {
-        Thread.sleep(SLEEP_DURATION_MS);
-        _totalSleepTime += SLEEP_DURATION_MS;
-      } catch (InterruptedException e) {
-        throw new IOException("Thread aborted", e);
-      }
-    }
-  }
-
-  /**
-   * Getter for the number of bytes read from this stream, since creation.
-   * 
-   * @return The number of bytes.
-   */
-  public long getTotalBytesRead() {
-    return _bytesRead;
-  }
-
-  /**
-   * Getter for the read-rate from this stream, since creation. Calculated as
-   * bytesRead/elapsedTimeSinceStart.
-   * 
-   * @return Read rate, in bytes/sec.
-   */
-  public double getBytesPerSec() {
-    double elapsed = (System.nanoTime() - _startTime) / 1000000000;
-    if (elapsed == 0) {
-      return _bytesRead;
-    } else {
-      return _bytesRead / elapsed;
-    }
-  }
-
-  /**
-   * Getter the total time spent in sleep.
-   * 
-   * @return Number of milliseconds spent in sleep.
-   */
-  public long getTotalSleepTime() {
-    return _totalSleepTime;
-  }
-
-  /** @inheritDoc */
-  @Override
-  public String toString() {
-    return "ThrottledIndexInput{" + "bytesRead=" + _bytesRead + ", maxBytesPerSec=" + _maxBytesPerSec
-        + ", bytesPerSec=" + getBytesPerSec() + ", totalSleepTime=" + _totalSleepTime + '}';
-  }
-
-  public long getStartTime() {
-    return _startTime;
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java
b/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java
deleted file mode 100644
index 7e092c3..0000000
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableDirectory.java
+++ /dev/null
@@ -1,152 +0,0 @@
-package org.apache.blur.lucene.warmup;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.blur.store.hdfs.DirectoryDecorator;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-
-public class TraceableDirectory extends Directory implements DirectoryDecorator {
-
-  private final Directory _dir;
-  /**
-   * One might feel compelled to make this field volatile because of the
-   * massively parallel environment that Blur operates. Please do NOT! This
-   * field cannot be volatile for performance reasons. For every query in Lucene
-   * there are a number of clones made of each file handle in each segment in
-   * each index. If this were volatile every thread would have to check main
-   * memory for this field every time a clone was called (see
-   * {@link TraceableIndexInput}). In fact I want this field to the remain false
-   * in all cases, except for when a warm up trace is needed. If that occurs the
-   * Thread running the actual the trace will set this field to true for a brief
-   * moment while all the clones are made and the trace is performed (the clones
-   * are also made with this this trace thread), this will likely just alter the
-   * value for that thread and not commit it back to main memory (heap). After
-   * the trace is complete is will return the field to false. During the time
-   * when the field is true, if any other field actually reads the value as true
-   * there is a second {@link ThreadLocal} field in {@link IndexWarmup} that
-   * prevents any other threads from running a trace. However there will be a
-   * small performance penalty while that situation occurs, because accessing a
-   * {@link ThreadLocal} field is fairly expensive.
-   * 
-   * In short please leave this the way it is, if you really want to change it
-   * please post a question on the mail list first.
-   */
-  private boolean _trace = false;
-  private ThreadLocal<IndexTracer> _indexTracer = new ThreadLocal<IndexTracer>();
-
-  public TraceableDirectory(Directory dir) {
-    _dir = dir;
-  }
-
-  public void trace(String name, long filePointer) {
-    if (_trace) {
-      if (_indexTracer != null) {
-        _indexTracer.get().trace(name, filePointer);
-      }
-    }
-  }
-
-  public IndexInput openInput(String name, IOContext context) throws IOException {
-    return new TraceableIndexInput(this, name, context, _dir.openInput(name, context));
-  }
-
-  public String[] listAll() throws IOException {
-    return _dir.listAll();
-  }
-
-  public boolean fileExists(String name) throws IOException {
-    return _dir.fileExists(name);
-  }
-
-  public void deleteFile(String name) throws IOException {
-    _dir.deleteFile(name);
-  }
-
-  public long fileLength(String name) throws IOException {
-    return _dir.fileLength(name);
-  }
-
-  public IndexOutput createOutput(String name, IOContext context) throws IOException {
-    return _dir.createOutput(name, context);
-  }
-
-  public void sync(Collection<String> names) throws IOException {
-    _dir.sync(names);
-  }
-
-  public void close() throws IOException {
-    _dir.close();
-  }
-
-  public boolean isTrace() {
-    return _trace;
-  }
-
-  public void setTrace(boolean trace) {
-    _trace = trace;
-  }
-
-  public void setIndexTracer(IndexTracer indexTracer) {
-    _indexTracer.set(indexTracer);
-  }
-
-  public Lock makeLock(String name) {
-    return _dir.makeLock(name);
-  }
-
-  public void clearLock(String name) throws IOException {
-    _dir.clearLock(name);
-  }
-
-  public void setLockFactory(LockFactory lockFactory) throws IOException {
-    _dir.setLockFactory(lockFactory);
-  }
-
-  public LockFactory getLockFactory() {
-    return _dir.getLockFactory();
-  }
-
-  public String getLockID() {
-    return _dir.getLockID();
-  }
-
-  public String toString() {
-    return _dir.toString();
-  }
-
-  public void copy(Directory to, String src, String dest, IOContext context) throws IOException
{
-    _dir.copy(to, src, dest, context);
-  }
-
-  public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException
{
-    return _dir.createSlicer(name, context);
-  }
-
-  @Override
-  public Directory getOriginalDirectory() {
-    return _dir;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
b/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
deleted file mode 100644
index 9ad3cda..0000000
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
+++ /dev/null
@@ -1,93 +0,0 @@
-package org.apache.blur.lucene.warmup;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-
-public class TraceableIndexInput extends IndexInput {
-
-  private final TraceableDirectory _traceableDirectory;
-  private final String _name;
-  private IndexInput _input;
-  private boolean _traceOn;
-
-  public TraceableIndexInput(TraceableDirectory traceableDirectory, String name, IOContext
context, IndexInput input) {
-    super("TraceableIndexInput(" + input.toString() + ")");
-    _traceableDirectory = traceableDirectory;
-    _input = input;
-    _traceOn = _traceableDirectory.isTrace();
-    _name = name;
-  }
-
-  @Override
-  public void close() throws IOException {
-    _input.close();
-  }
-
-  @Override
-  public long getFilePointer() {
-    return _input.getFilePointer();
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    _input.seek(pos);
-  }
-
-  @Override
-  public long length() {
-    return _input.length();
-  }
-
-  @Override
-  public byte readByte() throws IOException {
-    trace();
-    return _input.readByte();
-  }
-
-  private void trace() {
-    if (_traceOn) {
-      if (IndexWarmup.isRunTrace()) {
-        _traceableDirectory.trace(_name, _input.getFilePointer());
-      }
-    }
-  }
-
-  @Override
-  public void readBytes(byte[] b, int offset, int len) throws IOException {
-    trace();
-    _input.readBytes(b, offset, len);
-  }
-
-  @Override
-  public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException
{
-    trace();
-    _input.readBytes(b, offset, len, useBuffer);
-  }
-
-  @Override
-  public IndexInput clone() {
-    TraceableIndexInput clone = (TraceableIndexInput) super.clone();
-    clone._input = _input.clone();
-    clone._traceOn = _traceableDirectory.isTrace();
-    return clone;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java b/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java
deleted file mode 100644
index 2fcc969..0000000
--- a/blur-store/src/test/java/org/apache/blur/lucene/warmup/IndexWarmupTest.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.lucene.warmup;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.OpenBitSet;
-import org.apache.lucene.util.Version;
-import org.junit.Test;
-
-public class IndexWarmupTest {
-
-  private Random _random = new Random();
-  private int _numberOfFields = 10;
-
-  @Test
-  public void testIndexWarmup() throws IOException {
-    File file = new File("./target/tmp/indexwarmup-test");
-    Directory dir = FSDirectory.open(file);
-
-    Directory directory = new TraceableDirectory(new SlowAccessDirectory(dir));
-    IndexReader indexReader = getIndexReader(directory);
-    int maxSampleSize = 1000;
-    int blockSize = 8192;
-    long totalLookups = 0;
-    for (String s : directory.listAll()) {
-      if (s.endsWith(".pos") || s.endsWith(".doc") || s.endsWith(".tim")) {
-        long fileLength = directory.fileLength(s);
-        long maxHits = (long) Math.ceil(fileLength / (double) blockSize);
-        totalLookups += maxHits;
-        System.out.println("file [" + s + "] size [" + fileLength + "] maxhits [" + maxHits
+ "]");
-      }
-    }
-    AtomicBoolean stop = new AtomicBoolean();
-    AtomicBoolean isClosed = new AtomicBoolean();
-    SlowAccessDirectory._reads.set(0);
-    IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, Long.MAX_VALUE);
-    long t1 = System.nanoTime();
-    Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(indexReader,
"test");
-    long sampleReads = SlowAccessDirectory._reads.get();
-    SlowAccessDirectory._reads.set(0);
-    long t2 = System.nanoTime();
-    for (int i = 0; i < _numberOfFields; i++) {
-      indexWarmup.warmFile(indexReader, sampleIndex, "test" + i, "test");
-    }
-    long t3 = System.nanoTime();
-    System.out.println((t2 - t1) / 1000000.0 + " " + (t3 - t2) / 1000000.0);
-    System.out.println(totalLookups + " " + sampleReads + " " + SlowAccessDirectory._reads.get());
-  }
-
-  @Test
-  public void testIndexWarmupBitSet() throws IOException {
-    File file = new File("./target/tmp/indexwarmup-test");
-    Directory dir = FSDirectory.open(file);
-
-    Directory directory = new TraceableDirectory(new SlowAccessDirectory(dir));
-    IndexReader indexReader = getIndexReader(directory);
-    int maxSampleSize = 1000;
-    int blockSize = 8192;
-    // int blockSize = 1024 * 1024;
-    long totalLookups = 0;
-    for (String s : directory.listAll()) {
-      if (s.endsWith(".pos") || s.endsWith(".doc") || s.endsWith(".tim")) {
-        long fileLength = directory.fileLength(s);
-        long maxHits = (long) Math.ceil(fileLength / (double) blockSize);
-        totalLookups += maxHits;
-        System.out.println("file [" + s + "] size [" + fileLength + "] maxhits [" + maxHits
+ "]");
-      }
-    }
-    AtomicBoolean stop = new AtomicBoolean();
-    AtomicBoolean isClosed = new AtomicBoolean();
-    SlowAccessDirectory._reads.set(0);
-    IndexWarmup indexWarmup = new IndexWarmup(isClosed, stop, maxSampleSize, Long.MAX_VALUE);
-    long t1 = System.nanoTime();
-    Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(indexReader,
"test");
-    long sampleReads = SlowAccessDirectory._reads.get();
-    SlowAccessDirectory._reads.set(0);
-    long t2 = System.nanoTime();
-    Map<String, OpenBitSet> filePartsToWarm = new HashMap<String, OpenBitSet>();
-    for (int i = 0; i < _numberOfFields; i++) {
-      indexWarmup.getFilePositionsToWarm(indexReader, sampleIndex, "test" + i, "test", filePartsToWarm,
blockSize);
-    }
-    indexWarmup.warmFile(indexReader, filePartsToWarm, "test", blockSize, 1024 * 1024);
-    long t3 = System.nanoTime();
-
-    for (Entry<String, OpenBitSet> e : filePartsToWarm.entrySet()) {
-      OpenBitSet bitSet = e.getValue();
-      System.out.println(bitSet.length() + " " + bitSet.cardinality());
-    }
-
-    System.out.println((t2 - t1) / 1000000.0 + " " + (t3 - t2) / 1000000.0);
-    System.out.println(totalLookups + " " + sampleReads + " " + SlowAccessDirectory._reads.get());
-  }
-
-  private IndexReader getIndexReader(Directory directory) throws IOException {
-    if (!DirectoryReader.indexExists(directory)) {
-      long t1 = System.nanoTime();
-      populate(directory);
-      long t2 = System.nanoTime();
-      System.out.println((t2 - t1) / 1000000.0);
-    }
-    return DirectoryReader.open(directory);
-  }
-
-  private void populate(Directory directory) throws IOException {
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
-    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
-    mergePolicy.setUseCompoundFile(false);
-    IndexWriter writer = new IndexWriter(directory, conf);
-    addDocs(writer);
-    writer.close();
-  }
-
-  private void addDocs(IndexWriter writer) throws IOException {
-    for (int i = 0; i < 20000; i++) {
-      writer.addDocument(getDoc());
-    }
-  }
-
-  private Iterable<? extends IndexableField> getDoc() {
-    Document document = new Document();
-    document.add(new TextField(getFieldName(), getText(), Store.YES));
-    return document;
-  }
-
-  private String getText() {
-    StringBuilder builder = new StringBuilder();
-    for (int i = 0; i < 1000; i++) {
-      builder.append(getWord()).append(' ');
-    }
-    return builder.toString();
-  }
-
-  private String getWord() {
-    StringBuilder builder = new StringBuilder();
-    for (int i = 0; i < 10; i++) {
-      builder.append(getChar());
-    }
-    return builder.toString();
-  }
-
-  private char getChar() {
-    return (char) (_random.nextInt(26) + 'a');
-  }
-
-  private String getFieldName() {
-    return "test" + _random.nextInt(_numberOfFields);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
b/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
deleted file mode 100644
index f9aff08..0000000
--- a/blur-store/src/test/java/org/apache/blur/lucene/warmup/SlowAccessDirectory.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.blur.lucene.warmup;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-
-public class SlowAccessDirectory extends Directory {
-  private final Directory _directory;
-
-  public static AtomicLong _reads = new AtomicLong();
-
-  static class SlowAccessIndexInput extends BufferedIndexInput {
-
-    private IndexInput _indexInput;
-
-    protected SlowAccessIndexInput(IndexInput indexInput) {
-      super(indexInput.toString(), 8192);
-      _indexInput = indexInput;
-    }
-
-    private void delay() throws IOException {
-      synchronized (this) {
-        try {
-          wait(1);
-        } catch (InterruptedException e) {
-          throw new IOException(e);
-        }
-      }
-    }
-
-    public void close() throws IOException {
-      _indexInput.close();
-    }
-
-    public long length() {
-      return _indexInput.length();
-    }
-
-    @Override
-    public SlowAccessIndexInput clone() {
-      SlowAccessIndexInput clone = (SlowAccessIndexInput) super.clone();
-      clone._indexInput = _indexInput.clone();
-      return clone;
-    }
-
-    @Override
-    protected void readInternal(byte[] b, int offset, int length) throws IOException {
-      delay();
-      _reads.incrementAndGet();
-      long filePointer = getFilePointer();
-      _indexInput.seek(filePointer);
-      _indexInput.readBytes(b, offset, length);
-    }
-
-    @Override
-    protected void seekInternal(long pos) throws IOException {
-
-    }
-
-  }
-
-  public SlowAccessDirectory(Directory directory) {
-    _directory = directory;
-  }
-
-  public String[] listAll() throws IOException {
-    return _directory.listAll();
-  }
-
-  public boolean fileExists(String name) throws IOException {
-    return _directory.fileExists(name);
-  }
-
-  public void deleteFile(String name) throws IOException {
-    _directory.deleteFile(name);
-  }
-
-  public long fileLength(String name) throws IOException {
-    return _directory.fileLength(name);
-  }
-
-  public IndexOutput createOutput(String name, IOContext context) throws IOException {
-    return _directory.createOutput(name, context);
-  }
-
-  public void sync(Collection<String> names) throws IOException {
-    _directory.sync(names);
-  }
-
-  public IndexInput openInput(String name, IOContext context) throws IOException {
-    return new SlowAccessIndexInput(_directory.openInput(name, context));
-  }
-
-  public Lock makeLock(String name) {
-    return _directory.makeLock(name);
-  }
-
-  public void clearLock(String name) throws IOException {
-    _directory.clearLock(name);
-  }
-
-  public void close() throws IOException {
-    _directory.close();
-  }
-
-  public void setLockFactory(LockFactory lockFactory) throws IOException {
-    _directory.setLockFactory(lockFactory);
-  }
-
-  public LockFactory getLockFactory() {
-    return _directory.getLockFactory();
-  }
-
-  public String getLockID() {
-    return _directory.getLockID();
-  }
-
-  public void copy(Directory to, String src, String dest, IOContext context) throws IOException
{
-    _directory.copy(to, src, dest, context);
-  }
-
-  public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException
{
-    return _directory.createSlicer(name, context);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 97c5cda..43c0b83 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -78,17 +78,15 @@ public class BlurConstants {
   public static final String BLUR_MAX_HEAP_PER_ROW_FETCH = "blur.max.heap.per.row.fetch";
   public static final String BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST = "blur.max.records.per.row.fetch.request";
   public static final String BLUR_SHARD_READ_INTERCEPTOR = "blur.shard.read.interceptor";
+  public static final String BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT = "blur.shard.internal.search.thread.count";
 
   public static final String BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT = "blur.shard.server.thrift.thread.count";
   public static final String BLUR_SHARD_CACHE_MAX_TIMETOLIVE = "blur.shard.cache.max.timetolive";
   public static final String BLUR_SHARD_FILTER_CACHE_CLASS = "blur.shard.filter.cache.class";
-  public static final String BLUR_SHARD_INDEX_WARMUP_CLASS = "blur.shard.index.warmup.class";
   public static final String BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT = "blur.indexmanager.search.thread.count";
   public static final String BLUR_INDEXMANAGER_MUTATE_THREAD_COUNT = "blur.indexmanager.mutate.thread.count";
   public static final String BLUR_INDEXMANAGER_FACET_THREAD_COUNT = "blur.indexmanager.facet.thread.count";
   public static final String BLUR_SHARD_DATA_FETCH_THREAD_COUNT = "blur.shard.data.fetch.thread.count";
-  public static final String BLUR_SHARD_WARMUP_THREAD_COUNT = "blur.shard.warmup.thread.count";
-  public static final String BLUR_SHARD_INDEX_WARMUP_THROTTLE = "blur.shard.index.warmup.throttle";
   public static final String BLUR_MAX_CLAUSE_COUNT = "blur.max.clause.count";
   public static final String BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS = "blur.shard.cache.max.querycache.elements";
   public static final String BLUR_SHARD_OPENER_THREAD_COUNT = "blur.shard.opener.thread.count";
@@ -100,7 +98,6 @@ public class BlurConstants {
   public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes";
   public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread";
   public static final String BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS = "blur.shard.distributed.layout.factory.class";
-  public static final String BLUR_SHARD_WARMUP_DISABLED = "blur.shard.warmup.disabled";
   public static final String BLUR_SHARD_DEEP_PAGING_CACHE_SIZE = "blur.shard.deep.paging.cache.size";
 
   public static final String BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT = "blur.shard.block.cache.v2.read.cache.ext";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e9dce540/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 008649e..1fa82cf 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -57,6 +57,9 @@ blur.shard.command.driver.threads=16
 # The number of command worker threads.
 blur.shard.command.worker.threads=16
 
+# The number of internal lucene worker threads used to make the search call more parallel.
+blur.shard.internal.search.thread.count=16
+
 # The number of fetcher threads
 blur.shard.data.fetch.thread.count=8
 
@@ -84,15 +87,6 @@ blur.shard.cache.max.timetolive=60000
 # Default implementation of the blur cache filter, which is a pass through filter that does
nothing
 blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache
 
-# Globally disable index warmup.
-blur.shard.warmup.disabled=true
-
-# Default Blur index warmup class that warms the fields provided in the table descriptor
-blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup
-
-# Throttles the warmup to 30MB/s across all the warmup threads
-blur.shard.index.warmup.throttle=30000000
-
 # By default the v2 version of the block cache is enabled
 blur.shard.block.cache.version=v2
 
@@ -175,9 +169,6 @@ blur.indexmanager.mutate.thread.count=8
 # The number of thread used for parallel faceting in the index manager
 blur.indexmanager.facet.thread.count=8
 
-# Number of threads used for warming up the index
-blur.shard.warmup.thread.count=8
-
 # The fetch count per Lucene search, this fetches pointers to hits
 blur.shard.fetchcount=110
 


Mime
View raw message