incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Fixed BLUR-317
Date Tue, 10 Dec 2013 04:34:54 GMT
Fixed BLUR-317


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2f86c628
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2f86c628
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2f86c628

Branch: refs/heads/apache-blur-0.2
Commit: 2f86c628829b5cd260c0fc20d85f0a5790203426
Parents: 0e6d2d4
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Dec 9 23:28:30 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Dec 9 23:28:30 2013 -0500

----------------------------------------------------------------------
 .../blur/manager/writer/BlurNRTIndex.java       |   2 +-
 .../mapreduce/lib/ProgressableDirectory.java    |   4 +-
 .../apache/blur/lucene/codec/Blur022Codec.java  |  26 +++
 .../lucene/codec/Blur022StoredFieldsFormat.java |  12 +-
 .../blur/lucene/codec/CachedDecompressor.java   |   7 +-
 .../refcounter/DirectoryReferenceCounter.java   |   6 +-
 .../blur/lucene/warmup/ThrottledIndexInput.java | 224 +++++++++----------
 .../blur/lucene/warmup/TraceableIndexInput.java |   2 +-
 .../blur/store/blockcache/BlockDirectory.java   |   2 +-
 .../store/blockcache_v2/CacheIndexInput.java    |   2 +-
 .../apache/blur/store/hdfs/HdfsIndexInput.java  |   5 +-
 .../blur/lucene/codec/Blur022CodecTest.java     | 202 +++++++++++++++++
 .../org/apache/blur/utils/BlurConstants.java    |   7 +-
 .../src/main/resources/blur-default.properties  |   6 +
 docs/cluster-setup.html                         |   4 +-
 docs/using-blur.html                            |  12 +
 16 files changed, 387 insertions(+), 136 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 3909e5a..a74678b 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -105,7 +105,7 @@ public class BlurNRTIndex extends BlurIndex {
     Analyzer analyzer = fieldManager.getAnalyzerForIndex();
     IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer);
     conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
-    conf.setCodec(new Blur022Codec());
+    conf.setCodec(new Blur022Codec(_tableContext.getBlurConfiguration()));
     conf.setSimilarity(_tableContext.getSimilarity());
     AtomicBoolean stop = new AtomicBoolean();
     conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, stop, _isClosed));

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
index ef4dd78..004e1fa 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java
@@ -40,7 +40,7 @@ import org.apache.lucene.store.LockFactory;
 public class ProgressableDirectory extends Directory {
 
   private static final Log LOG = LogFactory.getLog(ProgressableDirectory.class);
-  
+
   private final Directory _directory;
   private final Progressable _progressable;
 
@@ -248,7 +248,7 @@ public class ProgressableDirectory extends Directory {
     private Progressable _progressable;
 
     ProgressableIndexInput(String name, IndexInput indexInput, int buffer, Progressable progressable) {
-      super(name, buffer);
+      super("ProgressableIndexInput(" + indexInput.toString() + ")", buffer);
       _indexInput = indexInput;
       _length = indexInput.length();
       _progressable = progressable;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java
index a4ebdcd..deba9ef 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java
@@ -16,6 +16,13 @@
  */
 package org.apache.blur.lucene.codec;
 
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_CHUNKSIZE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_COMPRESSIONMODE;
+import static org.apache.blur.utils.BlurConstants.FAST;
+import static org.apache.blur.utils.BlurConstants.FAST_DECOMPRESSION;
+import static org.apache.blur.utils.BlurConstants.HIGH_COMPRESSION;
+
+import org.apache.blur.BlurConfiguration;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
@@ -33,6 +40,7 @@ import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 
 public class Blur022Codec extends Codec {
+
   private final StoredFieldsFormat fieldsFormat;
   private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
   private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat();
@@ -63,6 +71,24 @@ public class Blur022Codec extends Codec {
     fieldsFormat = new Blur022StoredFieldsFormat(chunkSize, compressionMode);
   }
 
+  public Blur022Codec(BlurConfiguration configuration) {
+    this(configuration.getInt(BLUR_SHARD_INDEX_CHUNKSIZE, 1 << 14), getCompressionMode(configuration));
+  }
+
+  private static CompressionMode getCompressionMode(BlurConfiguration configuration) {
+    String type = configuration.get(BLUR_SHARD_INDEX_COMPRESSIONMODE, FAST);
+    if (HIGH_COMPRESSION.equals(type)) {
+      return CompressionMode.HIGH_COMPRESSION;
+    } else if (FAST.equals(type)) {
+      return CompressionMode.FAST;
+    } else if (FAST_DECOMPRESSION.equals(type)) {
+      return CompressionMode.FAST_DECOMPRESSION;
+    } else {
+      throw new IllegalArgumentException("blur.shard.index.compressionmode=" + type
+          + " not supported.  Valid entries are [FAST,FAST_DECOMPRESSION,HIGH_COMPRESSION]");
+    }
+  }
+
   @Override
   public final StoredFieldsFormat storedFieldsFormat() {
     return fieldsFormat;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
index e84b210..c43b0b2 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
@@ -16,6 +16,10 @@
  */
 package org.apache.blur.lucene.codec;
 
+import static org.apache.blur.utils.BlurConstants.FAST;
+import static org.apache.blur.utils.BlurConstants.FAST_DECOMPRESSION;
+import static org.apache.blur.utils.BlurConstants.HIGH_COMPRESSION;
+
 import java.io.IOException;
 
 import org.apache.lucene.codecs.StoredFieldsFormat;
@@ -30,14 +34,10 @@ import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-
 public final class Blur022StoredFieldsFormat extends StoredFieldsFormat {
 
   static final String STORED_FIELDS_FORMAT_CHUNK_SIZE = "StoredFieldsFormat.chunkSize";
   static final String STORED_FIELDS_FORMAT_COMPRESSION_MODE = "StoredFieldsFormat.compressionMode";
-  private static final String FAST_DECOMPRESSION = "FAST_DECOMPRESSION";
-  private static final String FAST = "FAST";
-  private static final String HIGH_COMPRESSION = "HIGH_COMPRESSION";
   private static final String FORMAT_NAME = "Blur022StoredFields";
   private static final String SEGMENT_SUFFIX = "";
   private final int _chunkSize;
@@ -52,7 +52,7 @@ public final class Blur022StoredFieldsFormat extends StoredFieldsFormat {
 
     final CompressionMode _compressionMode;
 
-    CachedCompressionMode(CompressionMode compressionMode) {
+    CachedCompressionMode(CompressionMode compressionMode, Directory directory, SegmentInfo si) {
       _compressionMode = compressionMode;
     }
 
@@ -76,7 +76,7 @@ public final class Blur022StoredFieldsFormat extends StoredFieldsFormat {
   @Override
   public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
       throws IOException {
-    CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si));
+    CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si), directory, si);
     return new CompressingStoredFieldsReader(directory, si, SEGMENT_SUFFIX, fn, context, FORMAT_NAME, compressionMode);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
index 10b2882..5483891 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
@@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRef;
 public class CachedDecompressor extends Decompressor {
 
   static class Entry {
-    IndexInput _indexInput;
+    String _inputName;
     long _position = -1;
     BytesRef _bytesRef = new BytesRef();
   }
@@ -47,13 +47,14 @@ public class CachedDecompressor extends Decompressor {
   public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException {
     if (in instanceof IndexInput) {
       IndexInput indexInput = (IndexInput) in;
+      String name = indexInput.toString();
       Entry entry = _cache.get();
       long filePointer = indexInput.getFilePointer();
       BytesRef cachedRef = entry._bytesRef;
-      if (entry._indexInput != indexInput || entry._position != filePointer) {
+      if (!name.equals(entry._inputName) || entry._position != filePointer) {
         cachedRef.grow(originalLength);
         _decompressor.decompress(in, originalLength, 0, originalLength, cachedRef);
-        entry._indexInput = indexInput;
+        entry._inputName = name;
         entry._position = filePointer;
       }
       bytes.copyBytes(cachedRef);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
index 9f0181c..cca20f3 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
@@ -51,7 +51,7 @@ public class DirectoryReferenceCounter extends Directory implements DirectoryDec
       counter = new AtomicInteger();
       _refCounters.put(name, counter);
     }
-    return new RefIndexInput(input.toString(), input, counter);
+    return new RefIndexInput(input, counter);
   }
 
   public void deleteFile(String name) throws IOException {
@@ -98,8 +98,8 @@ public class DirectoryReferenceCounter extends Directory implements DirectoryDec
     private IndexInput input;
     private AtomicInteger ref;
 
-    public RefIndexInput(String resourceDescription, IndexInput input, AtomicInteger ref) {
-      super(resourceDescription);
+    public RefIndexInput(IndexInput input, AtomicInteger ref) {
+      super("RefIndexInput(" + input.toString() + ")");
       this.input = input;
       this.ref = ref;
       ref.incrementAndGet();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
index 7b31b48..4a7f862 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/ThrottledIndexInput.java
@@ -34,118 +34,116 @@ import org.apache.lucene.store.IndexInput;
  */
 public class ThrottledIndexInput extends IndexInput {
 
-	private static final long SLEEP_DURATION_MS = 50;
-
-	private final IndexInput _rawStream;
-	private final double _maxBytesPerSec;
-	private final long _startTime = System.nanoTime();
-
-	private long _bytesRead = 0;
-	private long _totalSleepTime = 0;
-
-	public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec) {
-		super(rawStream.toString());
-		_rawStream = rawStream;
-		_maxBytesPerSec = maxBytesPerSec;
-	}
-
-	/** @inheritDoc */
-	@Override
-	public long getFilePointer() {
-		return _rawStream.getFilePointer();
-	}
-
-	/** @inheritDoc */
-	@Override
-	public void seek(long pos) throws IOException {
-		_rawStream.seek(pos);
-	}
-
-	/** @inheritDoc */
-	@Override
-	public long length() {
-		return _rawStream.length();
-	}
-
-	/** @inheritDoc */
-	@Override
-	public byte readByte() throws IOException {
-		throttle();
-		try {
-			return _rawStream.readByte();
-		} finally {
-			_bytesRead++;
-		}
-	}
-
-	/** @inheritDoc */
-	@Override
-	public void readBytes(byte[] b, int offset, int len) throws IOException {
-		throttle();
-		try {
-			_rawStream.readBytes(b, offset, len);
-		} finally {
-			_bytesRead += len;
-		}
-	}
-
-	/** @inheritDoc */
-	@Override
-	public void close() throws IOException {
-		_rawStream.close();
-	}
-
-	private void throttle() throws IOException {
-		while (getBytesPerSec() > _maxBytesPerSec) {
-			try {
-				Thread.sleep(SLEEP_DURATION_MS);
-				_totalSleepTime += SLEEP_DURATION_MS;
-			} catch (InterruptedException e) {
-				throw new IOException("Thread aborted", e);
-			}
-		}
-	}
-
-	/**
-	 * Getter for the number of bytes read from this stream, since creation.
-	 * 
-	 * @return The number of bytes.
-	 */
-	public long getTotalBytesRead() {
-		return _bytesRead;
-	}
-
-	/**
-	 * Getter for the read-rate from this stream, since creation. Calculated as
-	 * bytesRead/elapsedTimeSinceStart.
-	 * 
-	 * @return Read rate, in bytes/sec.
-	 */
-	public double getBytesPerSec() {
-		double elapsed = (System.nanoTime() - _startTime) / 1000000000;
-		if (elapsed == 0) {
-			return _bytesRead;
-		} else {
-			return _bytesRead / elapsed;
-		}
-	}
-
-	/**
-	 * Getter the total time spent in sleep.
-	 * 
-	 * @return Number of milliseconds spent in sleep.
-	 */
-	public long getTotalSleepTime() {
-		return _totalSleepTime;
-	}
-
-	/** @inheritDoc */
-	@Override
-	public String toString() {
-		return "ThrottledIndexInput{" + "bytesRead=" + _bytesRead
-				+ ", maxBytesPerSec=" + _maxBytesPerSec + ", bytesPerSec="
-				+ getBytesPerSec() + ", totalSleepTime=" + _totalSleepTime
-				+ '}';
-	}
+  private static final long SLEEP_DURATION_MS = 50;
+
+  private final IndexInput _rawStream;
+  private final double _maxBytesPerSec;
+  private final long _startTime = System.nanoTime();
+
+  private long _bytesRead = 0;
+  private long _totalSleepTime = 0;
+
+  public ThrottledIndexInput(IndexInput rawStream, long maxBytesPerSec) {
+    super("ThrottledIndexInput(" + rawStream.toString() + ")");
+    _rawStream = rawStream;
+    _maxBytesPerSec = maxBytesPerSec;
+  }
+
+  /** @inheritDoc */
+  @Override
+  public long getFilePointer() {
+    return _rawStream.getFilePointer();
+  }
+
+  /** @inheritDoc */
+  @Override
+  public void seek(long pos) throws IOException {
+    _rawStream.seek(pos);
+  }
+
+  /** @inheritDoc */
+  @Override
+  public long length() {
+    return _rawStream.length();
+  }
+
+  /** @inheritDoc */
+  @Override
+  public byte readByte() throws IOException {
+    throttle();
+    try {
+      return _rawStream.readByte();
+    } finally {
+      _bytesRead++;
+    }
+  }
+
+  /** @inheritDoc */
+  @Override
+  public void readBytes(byte[] b, int offset, int len) throws IOException {
+    throttle();
+    try {
+      _rawStream.readBytes(b, offset, len);
+    } finally {
+      _bytesRead += len;
+    }
+  }
+
+  /** @inheritDoc */
+  @Override
+  public void close() throws IOException {
+    _rawStream.close();
+  }
+
+  private void throttle() throws IOException {
+    while (getBytesPerSec() > _maxBytesPerSec) {
+      try {
+        Thread.sleep(SLEEP_DURATION_MS);
+        _totalSleepTime += SLEEP_DURATION_MS;
+      } catch (InterruptedException e) {
+        throw new IOException("Thread aborted", e);
+      }
+    }
+  }
+
+  /**
+   * Getter for the number of bytes read from this stream, since creation.
+   * 
+   * @return The number of bytes.
+   */
+  public long getTotalBytesRead() {
+    return _bytesRead;
+  }
+
+  /**
+   * Getter for the read-rate from this stream, since creation. Calculated as
+   * bytesRead/elapsedTimeSinceStart.
+   * 
+   * @return Read rate, in bytes/sec.
+   */
+  public double getBytesPerSec() {
+    double elapsed = (System.nanoTime() - _startTime) / 1000000000;
+    if (elapsed == 0) {
+      return _bytesRead;
+    } else {
+      return _bytesRead / elapsed;
+    }
+  }
+
+  /**
+   * Getter the total time spent in sleep.
+   * 
+   * @return Number of milliseconds spent in sleep.
+   */
+  public long getTotalSleepTime() {
+    return _totalSleepTime;
+  }
+
+  /** @inheritDoc */
+  @Override
+  public String toString() {
+    return "ThrottledIndexInput{" + "bytesRead=" + _bytesRead + ", maxBytesPerSec=" + _maxBytesPerSec
+        + ", bytesPerSec=" + getBytesPerSec() + ", totalSleepTime=" + _totalSleepTime + '}';
+  }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
index a62062d..9ad3cda 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/TraceableIndexInput.java
@@ -29,7 +29,7 @@ public class TraceableIndexInput extends IndexInput {
   private boolean _traceOn;
 
   public TraceableIndexInput(TraceableDirectory traceableDirectory, String name, IOContext context, IndexInput input) {
-    super(name);
+    super("TraceableIndexInput(" + input.toString() + ")");
     _traceableDirectory = traceableDirectory;
     _input = input;
     _traceOn = _traceableDirectory.isTrace();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java b/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
index 69a1320..46bbc6c 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache/BlockDirectory.java
@@ -137,7 +137,7 @@ public class BlockDirectory extends Directory implements DirectoryDecorator {
 
     public CachedIndexInput(IndexInput source, int blockSize, String name, String cacheName, Cache cache,
         IOContext context) {
-      super(name, context);
+      super("CachedIndexInput(" + source.toString() + ")", context);
       _source = source;
       _blockSize = blockSize;
       _fileLength = source.length();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
index 59d3775..54f1097 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexInput.java
@@ -46,7 +46,7 @@ public class CacheIndexInput extends IndexInput {
 
   public CacheIndexInput(CacheDirectory directory, String fileName, IndexInput indexInput, Cache cache)
       throws IOException {
-    super(fileName);
+    super("CacheIndexInput(" + indexInput.toString() + ")");
     _directory = directory;
     _fileName = fileName;
     _indexInput = indexInput;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
index 12aa5ee..2854320 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
@@ -35,7 +35,7 @@ public class HdfsIndexInput extends ReusedBufferedIndexInput {
 
   public HdfsIndexInput(String name, FSDataInputStream inputStream, long length, MetricsGroup metricsGroup,
       int readVersion, Path path) throws IOException {
-    super(name);
+    super("HdfsIndexInput(" + path.toString() + ")");
     _inputStream = inputStream;
     _length = length;
     _metricsGroup = metricsGroup;
@@ -55,7 +55,8 @@ public class HdfsIndexInput extends ReusedBufferedIndexInput {
 
   @Override
   protected void readInternal(byte[] b, int offset, int length) throws IOException {
-    Tracer trace = Trace.trace("filesystem - read", Trace.param("file", _path), Trace.param("location", getFilePointer()),Trace.param("length", length));
+    Tracer trace = Trace.trace("filesystem - read", Trace.param("file", _path),
+        Trace.param("location", getFilePointer()), Trace.param("length", length));
     try {
       long start = System.nanoTime();
       long filePointer = getFilePointer();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-store/src/test/java/org/apache/blur/lucene/codec/Blur022CodecTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/lucene/codec/Blur022CodecTest.java b/blur-store/src/test/java/org/apache/blur/lucene/codec/Blur022CodecTest.java
new file mode 100644
index 0000000..b8ae70e
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/lucene/codec/Blur022CodecTest.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.codec;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+public class Blur022CodecTest {
+
+  private static final int WORDS = 10000;
+
+  @Test
+  public void testLargeDocs() throws IOException {
+    Random random = new Random();
+    Iterable<? extends IndexableField> doc = getLargeDoc(random);
+    RAMDirectory directory = new RAMDirectory();
+    IndexWriterConfig conf1 = new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43));
+    conf1.setCodec(new Blur022Codec());
+    IndexWriter writer1 = new IndexWriter(directory, conf1);
+    writer1.addDocument(doc);
+    writer1.close();
+
+    DirectoryReader reader1 = DirectoryReader.open(directory);
+    int numDocs1 = reader1.numDocs();
+    assertEquals(1, numDocs1);
+
+    // for (int i = 0; i < numDocs1; i++) {
+    // System.out.println(reader1.document(i));
+    // }
+
+    IndexWriterConfig conf2 = new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43));
+    conf2.setCodec(new Blur022Codec(1 << 16, CompressionMode.HIGH_COMPRESSION));
+    IndexWriter writer2 = new IndexWriter(directory, conf2);
+    writer2.addDocument(doc);
+    writer2.close();
+
+    DirectoryReader reader2 = DirectoryReader.open(directory);
+    int numDocs2 = reader2.numDocs();
+    assertEquals(2, numDocs2);
+
+    for (int i = 0; i < 2; i++) {
+
+      long t1 = System.nanoTime();
+      Document document1 = reader1.document(0);
+      long t2 = System.nanoTime();
+      Document document2 = reader2.document(1);
+      long t3 = System.nanoTime();
+
+      System.out.println((t3 - t2) / 1000000.0);
+      System.out.println((t2 - t1) / 1000000.0);
+
+      System.out.println("doc1 " + document1.hashCode());
+      System.out.println("doc2 " + document2.hashCode());
+    }
+
+    // for (int i = 0; i < numDocs2; i++) {
+    // System.out.println(reader2.document(i));
+    // }
+
+    // long fileLength = directory.fileLength("_0.fdt");
+
+    for (String name : directory.listAll()) {
+      if (name.endsWith(".fdt")) {
+        System.out.println(name);
+        System.out.println(directory.fileLength(name));
+      }
+    }
+
+  }
+
+  @Test
+  public void testSmallDocs() throws IOException {
+
+    RAMDirectory directory = new RAMDirectory();
+    IndexWriterConfig conf1 = new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43));
+    conf1.setCodec(new Blur022Codec());
+    Random random1 = new Random(1);
+    IndexWriter writer1 = new IndexWriter(directory, conf1);
+    for (int i = 0; i < 1000; i++) {
+      writer1.addDocument(getSmallDoc(random1));
+    }
+    writer1.close();
+
+    DirectoryReader reader1 = DirectoryReader.open(directory);
+    int numDocs1 = reader1.numDocs();
+    assertEquals(1000, numDocs1);
+
+    // for (int i = 0; i < numDocs1; i++) {
+    // System.out.println(reader1.document(i));
+    // }
+
+    IndexWriterConfig conf2 = new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43));
+    conf2.setCodec(new Blur022Codec(1 << 16, CompressionMode.HIGH_COMPRESSION));
+    Random random2 = new Random(1);
+    IndexWriter writer2 = new IndexWriter(directory, conf2);
+    for (int i = 0; i < 1000; i++) {
+      writer2.addDocument(getSmallDoc(random2));
+    }
+    writer2.close();
+
+    DirectoryReader reader2 = DirectoryReader.open(directory);
+    int numDocs2 = reader2.numDocs();
+    assertEquals(2000, numDocs2);
+
+    for (int i = 0; i < 2; i++) {
+
+      long t1 = System.nanoTime();
+      long hash1 = 0;
+      long hash2 = 0;
+      for (int d = 0; d < 1000; d++) {
+        Document document1 = reader1.document(d);
+        hash1 += document1.hashCode();
+      }
+      long t2 = System.nanoTime();
+      for (int d = 0; d < 1000; d++) {
+        Document document2 = reader2.document(d + 1000);
+        hash2 += document2.hashCode();
+      }
+      long t3 = System.nanoTime();
+
+      System.out.println((t3 - t2) / 1000000.0);
+      System.out.println((t2 - t1) / 1000000.0);
+
+      System.out.println("doc1 " + hash1);
+      System.out.println("doc2 " + hash2);
+    }
+
+    // for (int i = 0; i < numDocs2; i++) {
+    // System.out.println(reader2.document(i));
+    // }
+
+    // long fileLength = directory.fileLength("_0.fdt");
+
+    for (String name : directory.listAll()) {
+      if (name.endsWith(".fdt")) {
+        System.out.println(name);
+        System.out.println(directory.fileLength(name));
+      }
+    }
+  }
+
+  private Iterable<? extends IndexableField> getSmallDoc(Random random) {
+    Document document = new Document();
+    document.add(new StringField("word", getWord(random), Store.YES));
+    return document;
+  }
+
+  private Iterable<? extends IndexableField> getLargeDoc(Random random) {
+    Document document = new Document();
+    String body = getBody(random);
+    // System.out.println("body size [" + body.length() + "]");
+    document.add(new TextField("body", body, Store.YES));
+    return document;
+  }
+
+  private String getBody(Random random) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < WORDS; i++) {
+      builder.append(getWord(random)).append(' ');
+    }
+    return builder.toString();
+  }
+
+  private String getWord(Random random) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < 20; i++) {
+      builder.append((char) (random.nextInt(26) + 'a'));
+    }
+    return builder.toString();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 513409a..221f96b 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -35,7 +35,12 @@ public class BlurConstants {
   public static final String DEFAULT_FAMILY = "_default_";
   public static final String SUPER = "super";
   public static final String SEP = ".";
-
+  
+  public static final String FAST_DECOMPRESSION = "FAST_DECOMPRESSION";
+  public static final String FAST = "FAST";
+  public static final String HIGH_COMPRESSION = "HIGH_COMPRESSION";
+  public static final String BLUR_SHARD_INDEX_CHUNKSIZE = "blur.shard.index.chunksize";
+  public static final String BLUR_SHARD_INDEX_COMPRESSIONMODE = "blur.shard.index.compressionmode";
   public static final String BLUR_SHARD_BALANCER_PERIOD = "blur.shard.balancer.period";
   public static final String BLUR_TABLE_PATH = "blur.table.path";
   public static final String BLUR_ZOOKEEPER_CONNECTION = "blur.zookeeper.connection";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 07f7193..a3407c9 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -124,6 +124,12 @@ blur.shard.block.cache.v2.write.nocache.ext=fdt
 # v2 version of block cache only. This specifies the default behavior if a file type is not specified in the cache or nocache lists during writes.  Values true | false
 blur.shard.block.cache.v2.write.default=true
 
+# Sets the compression of used in the storing of the fields. Valid entries are FAST FAST_DECOMPRESSION HIGH_COMPRESSION/
+blur.shard.index.compressionmode=FAST
+
+# Sets the chunksize of the compression in the storing of the fields. Larger values may produce smaller fdt files at the small cost of fetch performance.
+blur.shard.index.chunksize=16384
+
 # The amount of memory to be used by 8K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as the total memory allocated.  For example if you increased the "blur.shard.block.cache.v2.fileBufferSize" to 64K (65536) then this property should to "blur.shard.buffercache.65536".  You can also define as many of these properties as needed.
 blur.shard.buffercache.8192=67108864
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/docs/cluster-setup.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html
index 6912e73..8545382 100644
--- a/docs/cluster-setup.html
+++ b/docs/cluster-setup.html
@@ -132,7 +132,7 @@ blur.cluster.default.table.uri=hdfs://namenode/blur/tables</code>
 			<h4>Default Properties</h4>
 			<table class="table-bordered table-striped table-condensed">
 			<tr><td>Property</td><td>Description</td></tr>
-<tr><td>blur.zookeeper.timeout (90000)</td><td>The zookeeper session timeout</td></tr><tr><td>blur.query.max.results.fetch (1000)</td><td>The maximum number of results that can be fetched in a single request</td></tr><tr><td>blur.query.max.row.fetch (100)</td><td>The maximum number of rows that can be fetched in a single request</td></tr><tr><td>blur.query.max.record.fetch (1000)</td><td>The maximum number of records that can be fetched in a single request</td></tr><tr><td>blur.metrics.reporters</td><td>Setup metric reporter</td></tr>
+<tr><td>blur.zookeeper.timeout (90000)</td><td>The zookeeper session timeout</td></tr><tr><td>blur.zookeeper.trace.path (/blur/traces)</td><td>The path in ZooKeeper where the distributed traces will be stored, if blank trace output will be written to the log or the HDFS store.</td></tr><tr><td>blur.hdfs.trace.path</td><td>The path in HDFS where the distributed traces will be stored, if blank trace output will be written to the log or the ZooKeeper store.</td></tr><tr><td>blur.query.max.results.fetch (1000)</td><td>The maximum number of results that can be fetched in a single request</td></tr><tr><td>blur.query.max.row.fetch (100)</td><td>The maximum number of rows that can be fetched in a single request</td></tr><tr><td>blur.query.max.record.fetch (1000)</td><td>The maximum number of records that can be fetched in a single request</td></tr><tr><td>blur.metrics.reporters</td><td>Setup metric reporter</td></tr><tr><td>blur.thrift.max.frame.size (16384000)</td><td>Thrift max frame size
 </td></tr>
 			</table>
             <h3 id="general-hadoop">Hadoop</h3>
 <p>
@@ -204,7 +204,7 @@ Swap can kill java perform, you may want to consider disabling swap.</div>
 			<h4>Default Properties</h4>
 			<table class="table-bordered table-striped table-condensed">
 			<tr><td>Property</td><td>Description</td></tr>
-<tr><td>blur.shard.hostname</td><td>The hostname for the shard, if blank the hostname is automatically detected</td></tr><tr><td>blur.shard.bind.address (0.0.0.0)</td><td>The binding address of the shard</td></tr><tr><td>blur.shard.bind.port (40020)</td><td>The default binding port of the shard server</td></tr><tr><td>blur.shard.data.fetch.thread.count (8)</td><td>The number of fetcher threads</td></tr><tr><td>blur.shard.server.thrift.thread.count (8)</td><td>The number of the thrift threads</td></tr><tr><td>blur.shard.thrift.selector.threads (2)</td><td>The number of threads used for selector processing inside the thrift server.</td></tr><tr><td>blur.shard.thrift.max.read.buffer.bytes (9223372036854775807)</td><td>The maximum number of bytes used for reading requests in the thrift server.</td></tr><tr><td>blur.shard.thrift.accept.queue.size.per.thread (4)</td><td>The size of the blocking queue per selector thread for passing accepted connections to the selector thread.</td></tr><tr
 ><td>blur.shard.opener.thread.count (8)</td><td>The number of threads that are used for opening indexes</td></tr><tr><td>blur.shard.cache.max.querycache.elements (128)</td><td>The number of cached queries</td></tr><tr><td>blur.shard.cache.max.timetolive (60000)</td><td>The time to live for the cache query</td></tr><tr><td>blur.shard.filter.cache.class (org.apache.blur.manager.DefaultBlurFilterCache)</td><td>Default implementation of the blur cache filter, which is a pass through filter that does nothing</td></tr><tr><td>blur.shard.index.warmup.class (org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup)</td><td>Default Blur index warmup class that warms the fields provided in the table descriptor</td></tr><tr><td>blur.shard.index.warmup.throttle (30000000)</td><td>Throttles the warmup to 30MB/s across all the warmup threads</td></tr><tr><td>blur.shard.block.cache.version (v2)</td><td>By default the v2 version of the block cache is enabled</td></tr><tr><td>blur.shard.block.cach
 e.total.size</td><td>By default the total amount of memory block cache will use is -XX:MaxDirectMemorySize - 64 MiB</td></tr><tr><td>blur.shard.blockcache.direct.memory.allocation (true)</td><td>v1 version of block cache only. By default the block cache using off heap memory</td></tr><tr><td>blur.shard.blockcache.slab.count (-1)</td><td>v1 version of block cache only. The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through the MaxDirectoryMemorySize provided to the JVM</td></tr><tr><td>blur.shard.block.cache.v2.fileBufferSize (8192)</td><td>v2 version of block cache only. File buffer size, this is the buffer size used to read and write to data to HDFS.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize (8192)</td><td>v2 version of block cache only. The is the size of the blocks in the off heap cache, it is good practice to have this match 'blur.s
 hard.block.cache.v2.fileBufferSize'.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize.filter (33554432)</td><td>blur.shard.block.cache.v2.cacheBlockSize.<ext>=</td></tr><tr><td>blur.shard.block.cache.v2.store (OFF_HEAP)</td><td>v2 version of block cache only. This is used to control if the block are created on or off heap.  Values are OFF_HEAP | ON_HEAP</td></tr><tr><td>blur.shard.block.cache.v2.read.cache.ext</td><td>v2 version of block cache only. This specifies what file types should be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.nocache.ext (fdt)</td><td>v2 version of block cache only. This specifies what file types should NOT be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.default (true)</td><td>v2 version of block cache only. This specifies the default behavior if a file type is not specified in the cache or nocache lists during
  reads.  Values true | false</td></tr><tr><td>blur.shard.block.cache.v2.write.cache.ext</td><td>v2 version of block cache only. This specifies what file types should be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.nocache.ext (fdt)</td><td>v2 version of block cache only. This specifies what file types should NOT be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.default (true)</td><td>v2 version of block cache only. This specifies the default behavior if a file type is not specified in the cache or nocache lists during writes.  Values true | false</td></tr><tr><td>blur.shard.buffercache.8192 (67108864)</td><td>The amount of memory to be used by 8K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as the total memory allocated.  For example if you increased the "blur.shard
 .block.cache.v2.fileBufferSize" to 64K (65536) then this property should to "blur.shard.buffercache.65536".  You can also define as many of these properties as needed.</td></tr><tr><td>blur.shard.buffercache.1024 (8388608)</td><td>The amount of memory to be used by 1K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as the total memory allocated.</td></tr><tr><td>blur.shard.safemodedelay (5000)</td><td>The number of milliseconds to wait for the cluster to settle once changes have ceased</td></tr><tr><td>blur.shard.time.between.commits (30000)</td><td>The default time between index commits</td></tr><tr><td>blur.shard.time.between.refreshs (3000)</td><td>The default time between index refreshs</td></tr><tr><td>blur.shard.merge.thread.count (3)</td><td>The max number of threads used during index merges</td></tr><tr><td>blur.max.clause.count (1024)</td><td>The maximum
  number of clauses in a BooleanQuery</td></tr><tr><td>blur.indexmanager.search.thread.count (8)</td><td>The number of thread used for parallel searching in the index manager</td></tr><tr><td>blur.indexmanager.mutate.thread.count (8)</td><td>The number of thread used for parallel mutating in the index manager</td></tr><tr><td>blur.shard.internal.search.thread.count (8)</td><td>The number of threads used for parallel searching in the index searchers</td></tr><tr><td>blur.shard.warmup.thread.count (8)</td><td>Number of threads used for warming up the index</td></tr><tr><td>blur.shard.fetchcount (100)</td><td>The fetch count per Lucene search, this fetches pointers to hits</td></tr><tr><td>blur.max.heap.per.row.fetch (10000000)</td><td>Heap limit on row fetch, once this limit has been reached the request will return</td></tr><tr><td>blur.max.records.per.row.fetch.request (1000)</td><td>The maximum number of records in a single row fetch</td></tr><tr><td>blur.gui.shard.port (40090)</td><
 td>The http status page port for the shard server</td></tr><tr><td>blur.shard.filtered.server.class</td><td>To intercept the calls made to the shard server and perform server side changes to the calls extend org.apache.blur.server.FilteredBlurServer.</td></tr>
+<tr><td>blur.shard.hostname</td><td>The hostname for the shard, if blank the hostname is automatically detected</td></tr><tr><td>blur.shard.bind.address (0.0.0.0)</td><td>The binding address of the shard</td></tr><tr><td>blur.shard.bind.port (40020)</td><td>The default binding port of the shard server</td></tr><tr><td>blur.shard.data.fetch.thread.count (8)</td><td>The number of fetcher threads</td></tr><tr><td>blur.shard.server.thrift.thread.count (8)</td><td>The number of the thrift threads</td></tr><tr><td>blur.shard.thrift.selector.threads (2)</td><td>The number of threads used for selector processing inside the thrift server.</td></tr><tr><td>blur.shard.thrift.max.read.buffer.bytes (9223372036854775807)</td><td>The maximum number of bytes used for reading requests in the thrift server.</td></tr><tr><td>blur.shard.thrift.accept.queue.size.per.thread (4)</td><td>The size of the blocking queue per selector thread for passing accepted connections to the selector thread.</td></tr><tr
 ><td>blur.shard.opener.thread.count (8)</td><td>The number of threads that are used for opening indexes</td></tr><tr><td>blur.shard.cache.max.querycache.elements (128)</td><td>The number of cached queries</td></tr><tr><td>blur.shard.cache.max.timetolive (60000)</td><td>The time to live for the cache query</td></tr><tr><td>blur.shard.filter.cache.class (org.apache.blur.manager.DefaultBlurFilterCache)</td><td>Default implementation of the blur cache filter, which is a pass through filter that does nothing</td></tr><tr><td>blur.shard.index.warmup.class (org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup)</td><td>Default Blur index warmup class that warms the fields provided in the table descriptor</td></tr><tr><td>blur.shard.index.warmup.throttle (30000000)</td><td>Throttles the warmup to 30MB/s across all the warmup threads</td></tr><tr><td>blur.shard.block.cache.version (v2)</td><td>By default the v2 version of the block cache is enabled</td></tr><tr><td>blur.shard.block.cach
 e.total.size</td><td>By default the total amount of memory block cache will use is -XX:MaxDirectMemorySize - 64 MiB</td></tr><tr><td>blur.shard.blockcache.direct.memory.allocation (true)</td><td>v1 version of block cache only. By default the block cache using off heap memory</td></tr><tr><td>blur.shard.blockcache.slab.count (-1)</td><td>v1 version of block cache only. The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through the MaxDirectoryMemorySize provided to the JVM</td></tr><tr><td>blur.shard.block.cache.v2.fileBufferSize (8192)</td><td>v2 version of block cache only. File buffer size, this is the buffer size used to read and write to data to HDFS.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize (8192)</td><td>v2 version of block cache only. The is the size of the blocks in the off heap cache, it is good practice to have this match 'blur.s
 hard.block.cache.v2.fileBufferSize'.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize.filter (33554432)</td><td>blur.shard.block.cache.v2.cacheBlockSize.<ext>=</td></tr><tr><td>blur.shard.block.cache.v2.store (OFF_HEAP)</td><td>v2 version of block cache only. This is used to control if the block are created on or off heap.  Values are OFF_HEAP | ON_HEAP</td></tr><tr><td>blur.shard.block.cache.v2.read.cache.ext</td><td>v2 version of block cache only. This specifies what file types should be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.nocache.ext (fdt)</td><td>v2 version of block cache only. This specifies what file types should NOT be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.default (true)</td><td>v2 version of block cache only. This specifies the default behavior if a file type is not specified in the cache or nocache lists during
  reads.  Values true | false</td></tr><tr><td>blur.shard.block.cache.v2.write.cache.ext</td><td>v2 version of block cache only. This specifies what file types should be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.nocache.ext (fdt)</td><td>v2 version of block cache only. This specifies what file types should NOT be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.default (true)</td><td>v2 version of block cache only. This specifies the default behavior if a file type is not specified in the cache or nocache lists during writes.  Values true | false</td></tr><tr><td>blur.shard.index.compressionmode (FAST)</td><td>Sets the compression of used in the storing of the fields. Valid entries are FAST FAST_DECOMPRESSION HIGH_COMPRESSION/</td></tr><tr><td>blur.shard.index.chunksize (16384)</td><td>Sets the chunksize of the compression in the storing of the fields. Larger values may produce smaller fd
 t files at the small cost of fetch performance.</td></tr><tr><td>blur.shard.buffercache.8192 (67108864)</td><td>The amount of memory to be used by 8K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as the total memory allocated.  For example if you increased the "blur.shard.block.cache.v2.fileBufferSize" to 64K (65536) then this property should to "blur.shard.buffercache.65536".  You can also define as many of these properties as needed.</td></tr><tr><td>blur.shard.buffercache.1024 (8388608)</td><td>The amount of memory to be used by 1K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as the total memory allocated.</td></tr><tr><td>blur.shard.safemodedelay (5000)</td><td>The number of milliseconds to wait for the cluster to settle once changes 
 have ceased</td></tr><tr><td>blur.shard.time.between.commits (30000)</td><td>The default time between index commits</td></tr><tr><td>blur.shard.time.between.refreshs (3000)</td><td>The default time between index refreshs</td></tr><tr><td>blur.shard.merge.thread.count (3)</td><td>The max number of threads used during index merges</td></tr><tr><td>blur.shard.balancer.period (300000)</td><td>The time period in which the balancer will recalculate the shard distribution.  NOTE: This is time is per shard server so the checking may occur more often.</td></tr><tr><td>blur.max.clause.count (1024)</td><td>The maximum number of clauses in a BooleanQuery</td></tr><tr><td>blur.indexmanager.search.thread.count (8)</td><td>The number of thread used for parallel searching in the index manager</td></tr><tr><td>blur.indexmanager.mutate.thread.count (8)</td><td>The number of thread used for parallel mutating in the index manager</td></tr><tr><td>blur.shard.internal.search.thread.count (8)</td><td>The 
 number of threads used for parallel searching in the index searchers</td></tr><tr><td>blur.shard.warmup.thread.count (8)</td><td>Number of threads used for warming up the index</td></tr><tr><td>blur.shard.fetchcount (100)</td><td>The fetch count per Lucene search, this fetches pointers to hits</td></tr><tr><td>blur.max.heap.per.row.fetch (10000000)</td><td>Heap limit on row fetch, once this limit has been reached the request will return</td></tr><tr><td>blur.max.records.per.row.fetch.request (1000)</td><td>The maximum number of records in a single row fetch</td></tr><tr><td>blur.gui.shard.port (40090)</td><td>The http status page port for the shard server</td></tr><tr><td>blur.shard.filtered.server.class</td><td>To intercept the calls made to the shard server and perform server side changes to the calls extend org.apache.blur.server.FilteredBlurServer.</td></tr>
 			</table>
 
             <h3 id="shard-blur-env">blur-env.sh</h3>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f86c628/docs/using-blur.html
----------------------------------------------------------------------
diff --git a/docs/using-blur.html b/docs/using-blur.html
index b54f7d1..12a92ec 100644
--- a/docs/using-blur.html
+++ b/docs/using-blur.html
@@ -117,6 +117,9 @@
 <li><a href="#shell_command_reset">&nbsp;&nbsp;reset</a></li>
 <li><a href="#shell_command_user">&nbsp;&nbsp;user</a></li>
 <li><a href="#shell_command_whoami">&nbsp;&nbsp;whoami</a></li>
+<li><a href="#shell_command_trace">&nbsp;&nbsp;trace</a></li>
+<li><a href="#shell_command_trace-remove">&nbsp;&nbsp;trace-remove</a></li>
+<li><a href="#shell_command_trace-list">&nbsp;&nbsp;trace-list</a></li>
 </ul></li>
 
 				</ul>
@@ -479,6 +482,15 @@ The following rules are used when interacting with the shell:
 <h4 id="shell_command_whoami">whoami</h4>
 <p>Description: Print current user.<br/>
 <pre><code class="bash">whoami </code></pre></p>
+<h4 id="shell_command_trace">trace</h4>
+<p>Description: Toggle tracing of commands on/off.<br/>
+<pre><code class="bash">trace </code></pre></p>
+<h4 id="shell_command_trace-remove">trace-remove</h4>
+<p>Description: Delete trace by id.<br/>
+<pre><code class="bash">trace-remove &lt;trace id&gt;</code></pre></p>
+<h4 id="shell_command_trace-list">trace-list</h4>
+<p>Description: List traces by id.<br/>
+<pre><code class="bash">trace-list </code></pre></p>
 
           </section>
           <section>


Mime
View raw message