incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cr...@apache.org
Subject [05/30] git commit: Added a config to be able to set the number of merge threads per shard server.
Date Thu, 07 Nov 2013 02:41:28 GMT
Added a config to be able to set the number of merge threads per shard server.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/11d4e14a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/11d4e14a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/11d4e14a

Branch: refs/heads/blur-console-v2
Commit: 11d4e14a54a58500dc00df1cd9ba40e6e74a4e05
Parents: b963786
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Oct 19 20:39:40 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Oct 19 20:40:08 2013 -0400

----------------------------------------------------------------------
 .../apache/blur/manager/indexserver/DistributedIndexServer.java | 5 ++---
 .../org/apache/blur/manager/indexserver/LocalIndexServer.java   | 2 +-
 .../org/apache/blur/manager/writer/SharedMergeScheduler.java    | 3 +--
 .../main/java/org/apache/blur/thrift/ThriftBlurShardServer.java | 5 +++--
 .../org/apache/blur/manager/writer/BlurIndexReaderTest.java     | 2 +-
 .../java/org/apache/blur/manager/writer/BlurNRTIndexTest.java   | 2 +-
 .../src/main/java/org/apache/blur/utils/BlurConstants.java      | 1 +
 blur-util/src/main/resources/blur-default.properties            | 3 +++
 8 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index bf50852..ca826d1 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -120,7 +120,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
   public DistributedIndexServer(Configuration configuration, ZooKeeper zookeeper, ClusterStatus
clusterStatus,
       BlurIndexWarmup warmup, BlurFilterCache filterCache, BlockCacheDirectoryFactory blockCacheDirectoryFactory,
       DistributedLayoutFactory distributedLayoutFactory, String cluster, String nodeName,
long safeModeDelay,
-      int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads) throws KeeperException,
+      int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads, int maxMergeThreads)
throws KeeperException,
       InterruptedException {
     super(clusterStatus, configuration, nodeName, cluster);
     _closer = Closer.create();
@@ -148,8 +148,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     _gc = _closer.register(new DirectoryReferenceFileGC());
 
     // @TODO allow for configuration of these
-    _mergeScheduler = _closer.register(new SharedMergeScheduler());
-    
+    _mergeScheduler = _closer.register(new SharedMergeScheduler(maxMergeThreads));
     
     _refresher = _closer.register(new BlurIndexRefresher());
     _indexCloser = _closer.register(new BlurIndexCloser());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index f9652ce..5e42c23 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -69,7 +69,7 @@ public class LocalIndexServer extends AbstractIndexServer {
   public LocalIndexServer(TableDescriptor tableDescriptor) throws IOException {
     _closer = Closer.create();
     _tableContext = TableContext.create(tableDescriptor);
-    _mergeScheduler = _closer.register(new SharedMergeScheduler());
+    _mergeScheduler = _closer.register(new SharedMergeScheduler(3));
     _gc = _closer.register(new DirectoryReferenceFileGC());
     _searchExecutor = Executors.newCachedThreadPool();
     _closer.register(new CloseableExecutorService(_searchExecutor));

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java
index e53f464..6fe8205 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/SharedMergeScheduler.java
@@ -41,8 +41,7 @@ public class SharedMergeScheduler implements Runnable, Closeable {
   private AtomicBoolean _running = new AtomicBoolean(true);
   private ExecutorService _service;
 
-  public SharedMergeScheduler() {
-    int threads = 3;
+  public SharedMergeScheduler(int threads) {
     _service = Executors.newThreadPool(SHARED_MERGE_SCHEDULER, threads, false);
     for (int i = 0; i < threads; i++) {
       _service.submit(this);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index f02bd3e..ffed891 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -34,6 +34,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_THROTTLE;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_MERGE_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT;
@@ -190,10 +191,10 @@ public class ThriftBlurShardServer extends ThriftServer {
     int shardOpenerThreadCount = configuration.getInt(BLUR_SHARD_OPENER_THREAD_COUNT, 16);
     int internalSearchThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16);
     int warmupThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16);
-
+    int maxMergeThreads = configuration.getInt(BLUR_SHARD_MERGE_THREAD_COUNT, 3);
     final DistributedIndexServer indexServer = new DistributedIndexServer(config, zooKeeper,
clusterStatus,
         indexWarmup, filterCache, blockCacheDirectoryFactory, distributedLayoutFactory, cluster,
nodeName,
-        safeModeDelay, shardOpenerThreadCount, internalSearchThreads, warmupThreads);
+        safeModeDelay, shardOpenerThreadCount, internalSearchThreads, warmupThreads, maxMergeThreads);
 
     final IndexManager indexManager = new IndexManager();
     indexManager.setIndexServer(indexServer);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
index 4a63b75..5c80a9e 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
@@ -68,7 +68,7 @@ public class BlurIndexReaderTest {
     rm(base);
     base.mkdirs();
 
-    mergeScheduler = new SharedMergeScheduler();
+    mergeScheduler = new SharedMergeScheduler(1);
     gc = new DirectoryReferenceFileGC();
 
     configuration = new Configuration();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
index cafcf46..9800eab 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
@@ -70,7 +70,7 @@ public class BlurNRTIndexTest {
     rm(base);
     base.mkdirs();
 
-    mergeScheduler = new SharedMergeScheduler();
+    mergeScheduler = new SharedMergeScheduler(1);
     gc = new DirectoryReferenceFileGC();
 
     configuration = new Configuration();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index fb5a0e6..ebebdb5 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -69,6 +69,7 @@ public class BlurConstants {
   public static final String BLUR_MAX_CLAUSE_COUNT = "blur.max.clause.count";
   public static final String BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS = "blur.shard.cache.max.querycache.elements";
   public static final String BLUR_SHARD_OPENER_THREAD_COUNT = "blur.shard.opener.thread.count";
+  public static final String BLUR_SHARD_MERGE_THREAD_COUNT = "blur.shard.merge.thread.count";
   public static final String BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE = "blur.shard.index.deletion.policy.maxage";
   public static final String BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE = "blur.zookeeper.system.time.tolerance";
   public static final String BLUR_SHARD_INDEX_SIMILARITY = "blur.shard.index.similarity";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/11d4e14a/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 922dd6d..fdae2e7 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -96,6 +96,9 @@ blur.shard.time.between.commits=30000
 # The default time between index refreshs
 blur.shard.time.between.refreshs=3000
 
+# The max number of threads used during index merges
+blur.shard.merge.thread.count=3
+
 # The maximum number of clauses in a BooleanQuery
 blur.max.clause.count=1024
 


Mime
View raw message