incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixed BLUR-157
Date Wed, 26 Jun 2013 19:57:23 GMT
Updated Branches:
  refs/heads/master 969c2e171 -> 745397cec


Fixed BLUR-157


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/745397ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/745397ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/745397ce

Branch: refs/heads/master
Commit: 745397cec5a49272c628a4be995bf349805214a3
Parents: 969c2e1
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Jun 26 15:57:08 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Jun 26 15:57:08 2013 -0400

----------------------------------------------------------------------
 .../indexserver/DistributedIndexServer.java     | 45 +++++++++++++++-----
 .../blur/manager/writer/BlurNRTIndex.java       |  2 +-
 .../blur/thrift/ThriftBlurShardServer.java      |  4 ++
 .../org/apache/blur/utils/BlurConstants.java    |  2 +
 .../src/main/resources/blur-default.properties  |  2 +
 5 files changed, 43 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 85c1810..46033e5 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -135,6 +135,9 @@ public class DistributedIndexServer extends AbstractIndexServer {
   private AtomicLong _indexMemoryUsage = new AtomicLong();
   private BlurIndexRefresher _refresher;
   private BlurIndexCloser _indexCloser;
+  private int _internalSearchThreads;
+  private ExecutorService _warmupExecutor;
+  private int _warmupThreads;
 
   public static interface ReleaseReader {
     void release() throws IOException;
@@ -158,7 +161,8 @@ public class DistributedIndexServer extends AbstractIndexServer {
 
     // @TODO allow for configuration of these
     _mergeScheduler = new SharedMergeScheduler();
-    _searchExecutor = Executors.newThreadPool("internal-search", 16);
+    _searchExecutor = Executors.newThreadPool("internal-search", _internalSearchThreads);
+    _warmupExecutor = Executors.newThreadPool("warmup", _warmupThreads);
     _closer = new IndexInputCloser();
     _closer.init();
     _refresher = new BlurIndexRefresher();
@@ -372,7 +376,6 @@ public class DistributedIndexServer extends AbstractIndexServer {
   @Override
   public void close() {
     if (_running.get()) {
-
       _shardStateManager.close();
       _running.set(false);
       closeAllIndexes();
@@ -387,6 +390,8 @@ public class DistributedIndexServer extends AbstractIndexServer {
       _closer.close();
       _gc.close();
       _openerService.shutdownNow();
+      _searchExecutor.shutdownNow();
+      _warmupExecutor.shutdownNow();
     }
   }
 
@@ -478,20 +483,30 @@ public class DistributedIndexServer extends AbstractIndexServer {
     }
     _filterCache.opening(table, shard, index);
     TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, _cluster, table);
-    return warmUp(index, tableDescriptor, shard);
+    warmUp(index, tableDescriptor, shard);
+    return index;
   }
 
-  private BlurIndex warmUp(BlurIndex index, TableDescriptor table, String shard) throws IOException
{
-    final IndexSearcherClosable searcher = index.getIndexReader();
-    IndexReader reader = searcher.getIndexReader();
-    _warmup.warmBlurIndex(table, shard, reader, index.isClosed(), new ReleaseReader() {
+  private void warmUp(final BlurIndex index, final TableDescriptor table, final String shard)
throws IOException {
+    _warmupExecutor.submit(new Runnable() {
       @Override
-      public void release() throws IOException {
-        // this will allow for closing of index
-        searcher.close();
+      public void run() {
+        try {
+          final IndexSearcherClosable searcher = index.getIndexReader();
+          IndexReader reader = searcher.getIndexReader();
+          _warmup.warmBlurIndex(table, shard, reader, index.isClosed(), new ReleaseReader()
{
+            @Override
+            public void release() throws IOException {
+              // this will allow for closing of index
+              searcher.close();
+            }
+          });
+        } catch (Exception e) {
+          LOG.error("Unknown error while trying to warmup index [" + index + "]", e);
+        }
       }
     });
-    return index;
+
   }
 
   private synchronized Map<String, BlurIndex> openMissingShards(final String table,
Set<String> shardsToServe,
@@ -742,4 +757,12 @@ public class DistributedIndexServer extends AbstractIndexServer {
   public void setClusterName(String cluster) {
     _cluster = cluster;
   }
+
+  public void setInternalSearchThreads(int internalSearchThreads) {
+    _internalSearchThreads = internalSearchThreads;
+  }
+
+  public void setWarmupThreads(int warmupThreads) {
+    _warmupThreads = warmupThreads;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 4b0bb20..c53af2e 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -182,7 +182,7 @@ public class BlurNRTIndex extends BlurIndex {
       _refresher.close();
       try {
         _recorder.close();
-        _writer.close();
+        _writer.close(false);
         getNRTManager().close();
       } finally {
         _directory.close();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index cd3188c..1d89ea2 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -30,9 +30,11 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COU
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
@@ -187,6 +189,8 @@ public class ThriftBlurShardServer extends ThriftServer {
     indexServer.setConfiguration(config);
     indexServer.setNodeName(nodeName);
     indexServer.setShardOpenerThreadCount(configuration.getInt(BLUR_SHARD_OPENER_THREAD_COUNT,
16));
+    indexServer.setWarmupThreads(configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16));
+    indexServer.setInternalSearchThreads(configuration.getInt(BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT,
16));
     indexServer.setZookeeper(zooKeeper);
     indexServer.setFilterCache(filterCache);
     indexServer.setSafeModeDelay(configuration.getLong(BLUR_SHARD_SAFEMODEDELAY, 60000));

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 28ee666..de2bcf8 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -59,6 +59,8 @@ public class BlurConstants {
   public static final String BLUR_SHARD_INDEX_WARMUP_CLASS = "blur.shard.index.warmup.class";
   public static final String BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT = "blur.indexmanager.search.thread.count";
   public static final String BLUR_SHARD_DATA_FETCH_THREAD_COUNT = "blur.shard.data.fetch.thread.count";
+  public static final String BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT = "blur.shard.internal.search.thread.count";
+  public static final String BLUR_SHARD_WARMUP_THREAD_COUNT = "blur.shard.warmup.thread.count";
   public static final String BLUR_MAX_CLAUSE_COUNT = "blur.max.clause.count";
   public static final String BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS = "blur.shard.cache.max.querycache.elements";
   public static final String BLUR_SHARD_OPENER_THREAD_COUNT = "blur.shard.opener.thread.count";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/745397ce/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 9a703d8..9187a05 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -32,6 +32,8 @@ blur.shard.time.between.commits=30000
 blur.shard.time.between.refreshs=3000
 blur.max.clause.count=1024
 blur.indexmanager.search.thread.count=8
+blur.shard.internal.search.thread.count=8
+blur.shard.warmup.thread.count=8
 
 blur.controller.hostname=
 blur.controller.bind.address=0.0.0.0


Mime
View raw message