incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Adding another config for the length of the inmemory queue.
Date Fri, 14 Mar 2014 01:06:03 GMT
Adding another config for the length of the inmemory queue.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b32b1854
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b32b1854
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b32b1854

Branch: refs/heads/apache-blur-0.2
Commit: b32b1854bc54937c6d217f39100f1dd4ca0ebc73
Parents: cc599ec
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Mar 13 21:06:00 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Mar 13 21:06:00 2014 -0400

----------------------------------------------------------------------
 .../apache/blur/manager/writer/BlurIndexSimpleWriter.java    | 8 +++++---
 .../blur/manager/writer/BlurIndexSimpleWriterTest.java       | 3 +++
 .../src/main/java/org/apache/blur/utils/BlurConstants.java   | 1 +
 blur-util/src/main/resources/blur-default.properties         | 5 ++++-
 docs/cluster-setup.html                                      | 2 +-
 5 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b32b1854/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
index dee38c0..05b2d32 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
@@ -17,6 +17,7 @@
 package org.apache.blur.manager.writer;
 
 import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,6 +34,7 @@ import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
+import org.apache.blur.BlurConfiguration;
 import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.index.ExitableReader;
 import org.apache.blur.index.IndexDeletionPolicyReader;
@@ -73,7 +75,6 @@ public class BlurIndexSimpleWriter extends BlurIndex {
   private final AtomicReference<BlurIndexWriter> _writer = new AtomicReference<BlurIndexWriter>();
   private final boolean _makeReaderExitable = true;
   private IndexImporter _indexImporter;
-  // private ShardQueueReader _queueReader;
   private final ReentrantReadWriteLock _lock = new ReentrantReadWriteLock();
   private final WriteLock _writeLock = _lock.writeLock();
   private final ReadWriteLock _indexRefreshLock = new ReentrantReadWriteLock();
@@ -85,7 +86,7 @@ public class BlurIndexSimpleWriter extends BlurIndex {
   private final SnapshotIndexDeletionPolicy _snapshotIndexDeletionPolicy;
   private final String _context;
   private final AtomicInteger _writesWaiting = new AtomicInteger();
-  private final BlockingQueue<RowMutation> _queue = new ArrayBlockingQueue<RowMutation>(100);
+  private final BlockingQueue<RowMutation> _queue;
   private final MutationQueueProcessor _mutationQueueProcessor;
 
   public BlurIndexSimpleWriter(ShardContext shardContext, Directory directory, SharedMergeScheduler
mergeScheduler,
@@ -110,7 +111,8 @@ public class BlurIndexSimpleWriter extends BlurIndex {
         shardContext.getHdfsDirPath(), "generations"));
     _policy = new IndexDeletionPolicyReader(_snapshotIndexDeletionPolicy);
     _conf.setIndexDeletionPolicy(_policy);
-
+    BlurConfiguration blurConfiguration = _tableContext.getBlurConfiguration();
+    _queue = new ArrayBlockingQueue<RowMutation>(blurConfiguration.getInt(BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH,
100));
     _mutationQueueProcessor = new MutationQueueProcessor(_queue, this, _shardContext, _writesWaiting);
 
     if (!DirectoryReader.indexExists(directory)) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b32b1854/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java
index eefba67..4b9927c 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexSimpleWriterTest.java
@@ -108,6 +108,7 @@ public class BlurIndexSimpleWriterTest {
     tableProperties.put(BlurConstants.BLUR_SHARD_QUEUE_MAX_PAUSE_TIME_WHEN_EMPTY, "500");
     tableProperties.put(BlurConstants.BLUR_SHARD_QUEUE_MAX_QUEUE_BATCH_SIZE, "500");
     tableProperties.put(BlurConstants.BLUR_SHARD_QUEUE_MAX_WRITER_LOCK_TIME, "1000");
+    tableProperties.put(BlurConstants.BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH, "1000");
 
     tableDescriptor.setTableProperties(tableProperties);
     TableContext tableContext = TableContext.create(tableDescriptor);
@@ -298,6 +299,8 @@ public class BlurIndexSimpleWriterTest {
   public void testEnqueue() throws IOException, InterruptedException {
     setupWriter(_configuration);
     runQueueTest(TOTAL_ROWS_FOR_TESTS, TOTAL_ROWS_FOR_TESTS);
+    runQueueTest(TOTAL_ROWS_FOR_TESTS, TOTAL_ROWS_FOR_TESTS * 2);
+    runQueueTest(TOTAL_ROWS_FOR_TESTS, TOTAL_ROWS_FOR_TESTS * 3);
   }
 
   private void runQueueTest(final int mutatesToAdd, int numberOfValidDocs) throws IOException,
InterruptedException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b32b1854/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 0edb2ce..bf5a94d 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -40,6 +40,7 @@ public class BlurConstants {
   public static final String BLUR_SHARD_QUEUE_MAX_PAUSE_TIME_WHEN_EMPTY = "blur.shard.queue.max.pause.time.when.empty";
   public static final String BLUR_SHARD_QUEUE_MAX_WRITER_LOCK_TIME = "blur.shard.queue.max.writer.lock.time";
   public static final String BLUR_SHARD_QUEUE_MAX_QUEUE_BATCH_SIZE = "blur.shard.queue.max.queue.batch.size";
+  public static final String BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH = "blur.shard.queue.max.inmemory.length";
 
 //  public static final String BLUR_TABLE_INDEX_QUEUE_READER_CLASS = "blur.table.index.queue.reader.class";
 //  public static final String BLUR_TABLE_INDEX_QUEUE_READER_BACKOFF = "blur.table.index.queue.reader.backoff";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b32b1854/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 0dd0f19..8e82374 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -193,9 +193,12 @@ blur.shard.queue.max.pause.time.when.empty=1000
 # The maximum amount of time that the queue can lock the writer before committing.  NOTE:
Any other writer event will cause the queue to release it's lock on the writer.
 blur.shard.queue.max.writer.lock.time=5000
 
-# The maximum number of the RowMutations the writer can drain from the queue at a time. 
+# The maximum number of RowMutations the writer can drain from the queue at a time. 
 blur.shard.queue.max.queue.batch.size=100
 
+# The maximum number of RowMutations that can exist in the inmemory block queue any point
in time.  NOTE: This is PER SHARD.
+blur.shard.queue.max.inmemory.length=100
+
 
 ### Controller Server Configuration
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b32b1854/docs/cluster-setup.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html
index 177786e..ce14104 100644
--- a/docs/cluster-setup.html
+++ b/docs/cluster-setup.html
@@ -204,7 +204,7 @@ Swap can kill java perform, you may want to consider disabling swap.</div>
 			<h4>Default Properties</h4>
 			<table class="table-bordered table-striped table-condensed">
 			<tr><td>Property</td><td>Description</td></tr>
-<tr><td>blur.shard.hostname</td><td>The hostname for the shard, if
blank the hostname is automatically detected</td></tr><tr><td>blur.shard.bind.address
(0.0.0.0)</td><td>The binding address of the shard</td></tr><tr><td>blur.shard.bind.port
(40020)</td><td>The default binding port of the shard server</td></tr><tr><td>blur.shard.data.fetch.thread.count
(8)</td><td>The number of fetcher threads</td></tr><tr><td>blur.shard.server.thrift.thread.count
(8)</td><td>The number of the thrift threads</td></tr><tr><td>blur.shard.thrift.selector.threads
(2)</td><td>The number of threads used for selector processing inside the thrift
server.</td></tr><tr><td>blur.shard.thrift.max.read.buffer.bytes (9223372036854775807)</td><td>The
maximum number of bytes used for reading requests in the thrift server.</td></tr><tr><td>blur.shard.thrift.accept.queue.size.per.thread
(4)</td><td>The size of the blocking queue per selector thread for passing accepted
connections to the selector thread.</td></tr><tr
 ><td>blur.shard.opener.thread.count (8)</td><td>The number of threads
that are used for opening indexes</td></tr><tr><td>blur.shard.cache.max.querycache.elements
(128)</td><td>The number of cached queries</td></tr><tr><td>blur.shard.cache.max.timetolive
(60000)</td><td>The time to live for the cache query</td></tr><tr><td>blur.shard.filter.cache.class
(org.apache.blur.manager.DefaultBlurFilterCache)</td><td>Default implementation
of the blur cache filter, which is a pass through filter that does nothing</td></tr><tr><td>blur.shard.warmup.disabled
(false)</td><td>Globally disable index warmup.</td></tr><tr><td>blur.shard.index.warmup.class
(org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup)</td><td>Default Blur
index warmup class that warms the fields provided in the table descriptor</td></tr><tr><td>blur.shard.index.warmup.throttle
(30000000)</td><td>Throttles the warmup to 30MB/s across all the warmup threads</td></tr><tr><td>blur.shard.block.cache.version
(v2)</td><td>By 
 default the v2 version of the block cache is enabled</td></tr><tr><td>blur.shard.block.cache.total.size</td><td>By
default the total amount of memory block cache will use is -XX:MaxDirectMemorySize - 64 MiB</td></tr><tr><td>blur.shard.blockcache.direct.memory.allocation
(true)</td><td>v1 version of block cache only. By default the block cache using
off heap memory</td></tr><tr><td>blur.shard.blockcache.slab.count
(-1)</td><td>v1 version of block cache only. The slabs in the blockcache are automatically
configured by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through
the MaxDirectoryMemorySize provided to the JVM</td></tr><tr><td>blur.shard.block.cache.v2.fileBufferSize
(8192)</td><td>v2 version of block cache only. File buffer size, this is the buffer
size used to read and write to data to HDFS.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize
(8192)</td><td>v2 version of block cache only. The is t
 he size of the blocks in the off heap cache, it is good practice to have this match 'blur.shard.block.cache.v2.fileBufferSize'.
 For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize.filter
(33554432)</td><td>blur.shard.block.cache.v2.cacheBlockSize.<ext>=</td></tr><tr><td>blur.shard.block.cache.v2.store
(OFF_HEAP)</td><td>v2 version of block cache only. This is used to control if
the block are created on or off heap.  Values are OFF_HEAP | ON_HEAP</td></tr><tr><td>blur.shard.block.cache.v2.read.cache.ext</td><td>v2
version of block cache only. This specifies what file types should be cached during reads.
 Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.nocache.ext
(fdt)</td><td>v2 version of block cache only. This specifies what file types should
NOT be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.default
(true)</td><td>v2 version of block cache only. This specifie
 s the default behavior if a file type is not specified in the cache or nocache lists during
reads.  Values true | false</td></tr><tr><td>blur.shard.block.cache.v2.write.cache.ext</td><td>v2
version of block cache only. This specifies what file types should be cached during writes.
 Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.nocache.ext
(fdt)</td><td>v2 version of block cache only. This specifies what file types should
NOT be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.default
(true)</td><td>v2 version of block cache only. This specifies the default behavior
if a file type is not specified in the cache or nocache lists during writes.  Values true
| false</td></tr><tr><td>blur.shard.index.compressionmode (FAST)</td><td>Sets
the compression of used in the storing of the fields. Valid entries are FAST FAST_DECOMPRESSION
HIGH_COMPRESSION</td></tr><tr><td>blur.shard.index.chunksize (16384)</td><td>Sets
the chunk
 size of the compression in the storing of the fields. Larger values may produce smaller fdt
files at the small cost of fetch performance.</td></tr><tr><td>blur.shard.buffercache.8192
(67108864)</td><td>The amount of memory to be used by 8K byte buffers.  Note if
you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize"
you should adjust the buffer sizes as well as the total memory allocated.  For example if
you increased the "blur.shard.block.cache.v2.fileBufferSize" to 64K (65536) then this property
should to "blur.shard.buffercache.65536".  You can also define as many of these properties
as needed.</td></tr><tr><td>blur.shard.buffercache.1024 (8388608)</td><td>The
amount of memory to be used by 1K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize"
or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as
the total memory allocated.</td></tr><tr><td>blur.shard.safemodedelay
  (5000)</td><td>The number of milliseconds to wait for the cluster to settle
once changes have ceased</td></tr><tr><td>blur.shard.time.between.commits
(30000)</td><td>The default time between index commits</td></tr><tr><td>blur.shard.time.between.refreshs
(3000)</td><td>The default time between index refreshs</td></tr><tr><td>blur.shard.merge.thread.count
(8)</td><td>The max number of threads used during index merges</td></tr><tr><td>blur.max.clause.count
(1024)</td><td>The maximum number of clauses in a BooleanQuery</td></tr><tr><td>blur.indexmanager.search.thread.count
(8)</td><td>The number of thread used for parallel searching in the index manager</td></tr><tr><td>blur.indexmanager.mutate.thread.count
(8)</td><td>The number of thread used for parallel mutating in the index manager</td></tr><tr><td>blur.indexmanager.facet.thread.count
(8)</td><td>The number of thread used for parallel faceting in the index manager</td></tr><tr><td>blur.shard.warmup.thread.count
(8)</td><td>Number
  of threads used for warming up the index</td></tr><tr><td>blur.shard.fetchcount
(110)</td><td>The fetch count per Lucene search, this fetches pointers to hits</td></tr><tr><td>blur.max.heap.per.row.fetch
(10000000)</td><td>Heap limit on row fetch, once this limit has been reached the
request will return</td></tr><tr><td>blur.max.records.per.row.fetch.request
(1000)</td><td>The maximum number of records in a single row fetch</td></tr><tr><td>blur.gui.shard.port
(40090)</td><td>The http status page port for the shard server</td></tr><tr><td>blur.shard.filtered.server.class</td><td>To
intercept the calls made to the shard server and perform server side changes to the calls
extend org.apache.blur.server.FilteredBlurServer.</td></tr><tr><td>blur.shard.blurindex.class</td><td>Defines
the blur index class to be used to handle index requests.  This class has to extend org.apache.blur.manager.writer.BlurIndex.
 This can be defined globally as well as per table.</td></tr><tr><td>blur.shard.r
 ead.interceptor</td><td>Defines the blur read interceptor class that can mask
data from query results as well as data fetches.</td></tr><tr><td>blur.lucene.fst.bytearray.factory</td><td>Defines
the byte array factory class that blur will use to manage the FST trees in Lucene (extends
org.apache.blur.lucene.fst.ByteArrayFactory).</td></tr><tr><td>blur.shard.queue.max.pause.time.when.empty
(1000)</td><td>The maximum amount of time to pause before checking the queue for
RowMutations.</td></tr><tr><td>blur.shard.queue.max.writer.lock.time
(5000)</td><td>The maximum amount of time that the queue can lock the writer before
committing.  NOTE: Any other writer event will cause the queue to release it's lock on the
writer.</td></tr><tr><td>blur.shard.queue.max.queue.batch.size (100)</td><td>The
maximum number of the RowMutations the writer can drain from the queue at a time.</td></tr>
+<tr><td>blur.shard.hostname</td><td>The hostname for the shard, if
blank the hostname is automatically detected</td></tr><tr><td>blur.shard.bind.address
(0.0.0.0)</td><td>The binding address of the shard</td></tr><tr><td>blur.shard.bind.port
(40020)</td><td>The default binding port of the shard server</td></tr><tr><td>blur.shard.data.fetch.thread.count
(8)</td><td>The number of fetcher threads</td></tr><tr><td>blur.shard.server.thrift.thread.count
(8)</td><td>The number of the thrift threads</td></tr><tr><td>blur.shard.thrift.selector.threads
(2)</td><td>The number of threads used for selector processing inside the thrift
server.</td></tr><tr><td>blur.shard.thrift.max.read.buffer.bytes (9223372036854775807)</td><td>The
maximum number of bytes used for reading requests in the thrift server.</td></tr><tr><td>blur.shard.thrift.accept.queue.size.per.thread
(4)</td><td>The size of the blocking queue per selector thread for passing accepted
connections to the selector thread.</td></tr><tr
 ><td>blur.shard.opener.thread.count (8)</td><td>The number of threads
that are used for opening indexes</td></tr><tr><td>blur.shard.cache.max.querycache.elements
(128)</td><td>The number of cached queries</td></tr><tr><td>blur.shard.cache.max.timetolive
(60000)</td><td>The time to live for the cache query</td></tr><tr><td>blur.shard.filter.cache.class
(org.apache.blur.manager.DefaultBlurFilterCache)</td><td>Default implementation
of the blur cache filter, which is a pass through filter that does nothing</td></tr><tr><td>blur.shard.warmup.disabled
(false)</td><td>Globally disable index warmup.</td></tr><tr><td>blur.shard.index.warmup.class
(org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup)</td><td>Default Blur
index warmup class that warms the fields provided in the table descriptor</td></tr><tr><td>blur.shard.index.warmup.throttle
(30000000)</td><td>Throttles the warmup to 30MB/s across all the warmup threads</td></tr><tr><td>blur.shard.block.cache.version
(v2)</td><td>By 
 default the v2 version of the block cache is enabled</td></tr><tr><td>blur.shard.block.cache.total.size</td><td>By
default the total amount of memory block cache will use is -XX:MaxDirectMemorySize - 64 MiB</td></tr><tr><td>blur.shard.blockcache.direct.memory.allocation
(true)</td><td>v1 version of block cache only. By default the block cache using
off heap memory</td></tr><tr><td>blur.shard.blockcache.slab.count
(-1)</td><td>v1 version of block cache only. The slabs in the blockcache are automatically
configured by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through
the MaxDirectoryMemorySize provided to the JVM</td></tr><tr><td>blur.shard.block.cache.v2.fileBufferSize
(8192)</td><td>v2 version of block cache only. File buffer size, this is the buffer
size used to read and write to data to HDFS.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize
(8192)</td><td>v2 version of block cache only. The is t
 he size of the blocks in the off heap cache, it is good practice to have this match 'blur.shard.block.cache.v2.fileBufferSize'.
 For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize.filter
(33554432)</td><td>blur.shard.block.cache.v2.cacheBlockSize.<ext>=</td></tr><tr><td>blur.shard.block.cache.v2.store
(OFF_HEAP)</td><td>v2 version of block cache only. This is used to control if
the block are created on or off heap.  Values are OFF_HEAP | ON_HEAP</td></tr><tr><td>blur.shard.block.cache.v2.read.cache.ext</td><td>v2
version of block cache only. This specifies what file types should be cached during reads.
 Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.nocache.ext
(fdt)</td><td>v2 version of block cache only. This specifies what file types should
NOT be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.default
(true)</td><td>v2 version of block cache only. This specifie
 s the default behavior if a file type is not specified in the cache or nocache lists during
reads.  Values true | false</td></tr><tr><td>blur.shard.block.cache.v2.write.cache.ext</td><td>v2
version of block cache only. This specifies what file types should be cached during writes.
 Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.nocache.ext
(fdt)</td><td>v2 version of block cache only. This specifies what file types should
NOT be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.default
(true)</td><td>v2 version of block cache only. This specifies the default behavior
if a file type is not specified in the cache or nocache lists during writes.  Values true
| false</td></tr><tr><td>blur.shard.index.compressionmode (FAST)</td><td>Sets
the compression of used in the storing of the fields. Valid entries are FAST FAST_DECOMPRESSION
HIGH_COMPRESSION</td></tr><tr><td>blur.shard.index.chunksize (16384)</td><td>Sets
the chunk
 size of the compression in the storing of the fields. Larger values may produce smaller fdt
files at the small cost of fetch performance.</td></tr><tr><td>blur.shard.buffercache.8192
(67108864)</td><td>The amount of memory to be used by 8K byte buffers.  Note if
you change the "blur.shard.block.cache.v2.cacheBlockSize" or "blur.shard.block.cache.v2.fileBufferSize"
you should adjust the buffer sizes as well as the total memory allocated.  For example if
you increased the "blur.shard.block.cache.v2.fileBufferSize" to 64K (65536) then this property
should to "blur.shard.buffercache.65536".  You can also define as many of these properties
as needed.</td></tr><tr><td>blur.shard.buffercache.1024 (8388608)</td><td>The
amount of memory to be used by 1K byte buffers.  Note if you change the "blur.shard.block.cache.v2.cacheBlockSize"
or "blur.shard.block.cache.v2.fileBufferSize" you should adjust the buffer sizes as well as
the total memory allocated.</td></tr><tr><td>blur.shard.safemodedelay
  (5000)</td><td>The number of milliseconds to wait for the cluster to settle
once changes have ceased</td></tr><tr><td>blur.shard.time.between.commits
(30000)</td><td>The default time between index commits</td></tr><tr><td>blur.shard.time.between.refreshs
(3000)</td><td>The default time between index refreshs</td></tr><tr><td>blur.shard.merge.thread.count
(8)</td><td>The max number of threads used during index merges</td></tr><tr><td>blur.max.clause.count
(1024)</td><td>The maximum number of clauses in a BooleanQuery</td></tr><tr><td>blur.indexmanager.search.thread.count
(8)</td><td>The number of thread used for parallel searching in the index manager</td></tr><tr><td>blur.indexmanager.mutate.thread.count
(8)</td><td>The number of thread used for parallel mutating in the index manager</td></tr><tr><td>blur.indexmanager.facet.thread.count
(8)</td><td>The number of thread used for parallel faceting in the index manager</td></tr><tr><td>blur.shard.warmup.thread.count
(8)</td><td>Number
  of threads used for warming up the index</td></tr><tr><td>blur.shard.fetchcount
(110)</td><td>The fetch count per Lucene search, this fetches pointers to hits</td></tr><tr><td>blur.max.heap.per.row.fetch
(10000000)</td><td>Heap limit on row fetch, once this limit has been reached the
request will return</td></tr><tr><td>blur.max.records.per.row.fetch.request
(1000)</td><td>The maximum number of records in a single row fetch</td></tr><tr><td>blur.gui.shard.port
(40090)</td><td>The http status page port for the shard server</td></tr><tr><td>blur.shard.filtered.server.class</td><td>To
intercept the calls made to the shard server and perform server side changes to the calls
extend org.apache.blur.server.FilteredBlurServer.</td></tr><tr><td>blur.shard.blurindex.class</td><td>Defines
the blur index class to be used to handle index requests.  This class has to extend org.apache.blur.manager.writer.BlurIndex.
 This can be defined globally as well as per table.</td></tr><tr><td>blur.shard.r
 ead.interceptor</td><td>Defines the blur read interceptor class that can mask
data from query results as well as data fetches.</td></tr><tr><td>blur.lucene.fst.bytearray.factory</td><td>Defines
the byte array factory class that blur will use to manage the FST trees in Lucene (extends
org.apache.blur.lucene.fst.ByteArrayFactory).</td></tr><tr><td>blur.shard.queue.max.pause.time.when.empty
(1000)</td><td>The maximum amount of time to pause before checking the queue for
RowMutations.</td></tr><tr><td>blur.shard.queue.max.writer.lock.time
(5000)</td><td>The maximum amount of time that the queue can lock the writer before
committing.  NOTE: Any other writer event will cause the queue to release it's lock on the
writer.</td></tr><tr><td>blur.shard.queue.max.queue.batch.size (100)</td><td>The
maximum number of RowMutations the writer can drain from the queue at a time.</td></tr><tr><td>blur.shard.queue.max.inmemory.length
(100)</td><td>The maximum number of RowMutations that can exist in t
 he inmemory block queue any point in time.  NOTE: This is PER SHARD.</td></tr>
 			</table>
 
             <h3 id="shard-blur-env">blur-env.sh</h3>


Mime
View raw message