incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixed BLUR-254
Date Mon, 07 Oct 2013 13:35:21 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 5d159cb4f -> dc5a192fe


Fixed BLUR-254


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/dc5a192f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/dc5a192f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/dc5a192f

Branch: refs/heads/apache-blur-0.2
Commit: dc5a192fe4b7c49f6a4844cf3c9c93bfb692a479
Parents: 5d159cb
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Oct 7 09:34:53 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Oct 7 09:34:53 2013 -0400

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurControllerServer.java |  6 ++++
 .../blur/thrift/ThriftBlurShardServer.java      |  6 ++++
 .../org/apache/blur/thrift/ThriftServer.java    | 33 ++++++++++++++++++++
 .../org/apache/blur/utils/BlurConstants.java    |  6 ++++
 .../src/main/resources/blur-default.properties  | 20 ++++++++++++
 docs/cluster-setup.html                         | 18 +++++++++++
 6 files changed, 89 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index 4669bc8..347f1ca 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -32,6 +32,9 @@ import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_RETRY_MUTATE_D
 import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SERVER_REMOTE_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SERVER_THRIFT_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_SHARD_CONNECTION_TIMEOUT;
+import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD;
+import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES;
+import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST;
@@ -142,6 +145,9 @@ public class ThriftBlurControllerServer extends ThriftServer {
     server.setThreadCount(threadCount);
     server.setEventHandler(eventHandler);
     server.setIface(iface);
+    server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD,
4));
+    server.setMaxReadBufferBytes(configuration.getLong(BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES,
Long.MAX_VALUE));
+    server.setSelectorThreads(configuration.getInt(BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS,
2));
 
     int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_CONTROLLER_PORT));
     final HttpJettyServer httpServer;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 4c7d133..9febe8b 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -40,6 +40,9 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INTERNAL_SEARCH_THR
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES;
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_SELECTOR_THREADS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
@@ -275,6 +278,9 @@ public class ThriftBlurShardServer extends ThriftServer {
     server.setThreadCount(threadCount);
     server.setIface(iface);
     server.setEventHandler(eventHandler);
+    server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD,
4));
+    server.setMaxReadBufferBytes(configuration.getLong(BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES,
Long.MAX_VALUE));
+    server.setSelectorThreads(configuration.getInt(BLUR_SHARD_THRIFT_SELECTOR_THREADS, 2));
 
     // This will shutdown the server when the correct path is set in zk
     BlurShutdown shutdown = new BlurShutdown() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
index 130fd86..81423cd 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
@@ -52,6 +52,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException;
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.server.TThreadedSelectorServer;
+import org.apache.blur.thrift.server.TThreadedSelectorServer.Args.AcceptPolicy;
 
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Gauge;
@@ -72,6 +73,9 @@ public class ThriftServer {
   private ExecutorService _mutateExecutorService;
   private TServerEventHandler _eventHandler;
   private TNonblockingServerSocket _serverTransport;
+  private int _acceptQueueSizePerThread = 4;
+  private long _maxReadBufferBytes = Long.MAX_VALUE;
+  private int _selectorThreads = 2;
 
   public TNonblockingServerSocket getServerTransport() {
     return _serverTransport;
@@ -193,6 +197,11 @@ public class ThriftServer {
     args.executorService(_executorService);
     args.transportFactory(new TFramedTransport.Factory());
     args.protocolFactory(new TBinaryProtocol.Factory(true, true));
+    args.selectorThreads = _selectorThreads;
+    args.maxReadBufferBytes = _maxReadBufferBytes;
+    args.acceptQueueSizePerThread(_acceptQueueSizePerThread);
+    args.acceptPolicy(AcceptPolicy.FAIR_ACCEPT);
+
     _server = new TThreadedSelectorServer(args);
     _server.setServerEventHandler(_eventHandler);
     LOG.info("Starting server [{0}]", _nodeName);
@@ -284,4 +293,28 @@ public class ThriftServer {
     _eventHandler = eventHandler;
   }
 
+
+  public int getAcceptQueueSizePerThread() {
+    return _acceptQueueSizePerThread;
+  }
+
+  public void setAcceptQueueSizePerThread(int acceptQueueSizePerThread) {
+    _acceptQueueSizePerThread = acceptQueueSizePerThread;
+  }
+
+  public long getMaxReadBufferBytes() {
+    return _maxReadBufferBytes;
+  }
+
+  public void setMaxReadBufferBytes(long maxReadBufferBytes) {
+    _maxReadBufferBytes = maxReadBufferBytes;
+  }
+
+  public int getSelectorThreads() {
+    return _selectorThreads;
+  }
+
+  public void setSelectorThreads(int selectorThreads) {
+    _selectorThreads = selectorThreads;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 888c72f..2176ec6 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -72,6 +72,9 @@ public class BlurConstants {
   public static final String BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE = "blur.shard.index.deletion.policy.maxage";
   public static final String BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE = "blur.zookeeper.system.time.tolerance";
   public static final String BLUR_SHARD_INDEX_SIMILARITY = "blur.shard.index.similarity";
+  public static final String BLUR_SHARD_THRIFT_SELECTOR_THREADS = "blur.shard.thrift.selector.threads";
+  public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes";
+  public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread";
   
   public static final String BLUR_FIELDTYPE = "blur.fieldtype.";
 
@@ -92,6 +95,9 @@ public class BlurConstants {
   public static final String BLUR_CONTROLLER_RETRY_MAX_MUTATE_DELAY = "blur.controller.retry.max.mutate.delay";
   public static final String BLUR_CONTROLLER_RETRY_MAX_DEFAULT_DELAY = "blur.controller.retry.max.default.delay";
   public static final String BLUR_CONTROLLER_RETRY_MAX_FETCH_RETRIES = "blur.controller.retry.max.fetch.retries";
+  public static final String BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS = "blur.controller.thrift.selector.threads";
+  public static final String BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES = "blur.controller.thrift.max.read.buffer.bytes";
+  public static final String BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.controller.thrift.accept.queue.size.per.thread";
 
   public static final String BLUR_GUI_CONTROLLER_PORT = "blur.gui.controller.port";
   public static final String BLUR_GUI_SHARD_PORT = "blur.gui.shard.port";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index b96c6d9..48aaabf 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -45,6 +45,15 @@ blur.shard.data.fetch.thread.count=8
 # The number of the thrift threads
 blur.shard.server.thrift.thread.count=8
 
+# The number of threads used for selector processing inside the thrift server.
+blur.shard.thrift.selector.threads=2
+
+# The maximum number of bytes used for reading requests in the thrift server.
+blur.shard.thrift.max.read.buffer.bytes=9223372036854775807
+
+# The size of the blocking queue per selector thread for passing accepted connections to
the selector thread.
+blur.shard.thrift.accept.queue.size.per.thread=4
+
 # The number of threads that are used for opening indexes
 blur.shard.opener.thread.count=8
 
@@ -116,6 +125,8 @@ blur.max.records.per.row.fetch.request=1000
 blur.gui.shard.port=40090
 
 
+
+
 # Controller Properties
 
 # Sets the hostname for the controller, if blank the hostname is automatically detected
@@ -136,6 +147,15 @@ blur.controller.server.thrift.thread.count=32
 # The number of threads used for remote thrift requests to the shards server.  This should
be a large number.
 blur.controller.server.remote.thread.count=64
 
+# The number of threads used for selector processing inside the thrift server.
+blur.controller.thrift.selector.threads=2
+
+# The maximum number of bytes used for reading requests in the thrift server.
+blur.controller.thrift.max.read.buffer.bytes=9223372036854775807
+
+# The size of the blocking queue per selector thread for passing accepted connections to
the selector thread.
+blur.controller.thrift.accept.queue.size.per.thread=4
+
 # The number of hits to fetch per request to the shard servers
 blur.controller.remote.fetch.count=100
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dc5a192f/docs/cluster-setup.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html
index 4ef9e00..a2285dd 100644
--- a/docs/cluster-setup.html
+++ b/docs/cluster-setup.html
@@ -165,6 +165,15 @@ blur.controller.server.thrift.thread.count=32
 # the shards server.  This should be a large number.
 blur.controller.server.remote.thread.count=64
 
+# The number of threads used for selector processing inside the thrift server.
+blur.controller.thrift.selector.threads=2
+
+# The maximum number of bytes used for reading requests in the thrift server.
+blur.controller.thrift.max.read.buffer.bytes=9223372036854775807
+
+# The size of the blocking queue per selector thread for passing accepted connections to
the selector thread.
+blur.controller.thrift.accept.queue.size.per.thread=4
+
 # The number of hits to fetch per request to the shard servers
 blur.controller.remote.fetch.count=100
 
@@ -264,6 +273,15 @@ blur.shard.data.fetch.thread.count=8
 # The number of the thrift threads
 blur.shard.server.thrift.thread.count=8
 
+# The number of threads used for selector processing inside the thrift server.
+blur.shard.thrift.selector.threads=2
+
+# The maximum number of bytes used for reading requests in the thrift server.
+blur.shard.thrift.max.read.buffer.bytes=9223372036854775807
+
+# The size of the blocking queue per selector thread for passing accepted connections to
the selector thread.
+blur.shard.thrift.accept.queue.size.per.thread=4
+
 # The number of threads that are used for opening indexes
 blur.shard.opener.thread.count=8
 


Mime
View raw message