incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Fixed BLUR-314
Date Tue, 03 Dec 2013 18:17:09 GMT
Fixed BLUR-314


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a348d49c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a348d49c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a348d49c

Branch: refs/heads/apache-blur-0.2
Commit: a348d49c2bb7b07e0cf1881bdecd253526cf354f
Parents: deef456
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Dec 3 13:16:03 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Dec 3 13:16:03 2013 -0500

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurControllerServer.java       |  2 ++
 .../org/apache/blur/thrift/ThriftBlurShardServer.java |  4 +++-
 .../java/org/apache/blur/thrift/ThriftServer.java     | 14 ++++++++++++--
 .../main/java/org/apache/blur/thrift/ClientPool.java  | 11 ++++++++---
 .../java/org/apache/blur/utils/BlurConstants.java     |  2 ++
 blur-util/src/main/resources/blur-default.properties  |  3 +++
 6 files changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index 2d78a13..b796f2b 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -39,6 +39,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT;
+import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
@@ -164,6 +165,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
     server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD,
4));
     server.setMaxReadBufferBytes(configuration.getLong(BLUR_CONTROLLER_THRIFT_MAX_READ_BUFFER_BYTES,
Long.MAX_VALUE));
     server.setSelectorThreads(configuration.getInt(BLUR_CONTROLLER_THRIFT_SELECTOR_THREADS,
2));
+    server.setMaxFrameSize(configuration.getInt(BLUR_THRIFT_MAX_FRAME_SIZE, 16384000));
 
     int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_CONTROLLER_PORT));
     final HttpJettyServer httpServer;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index d247d3a..4159b91 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -43,6 +43,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_ACCEPT_QUEUE
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_THRIFT_SELECTOR_THREADS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_WARMUP_THREAD_COUNT;
+import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT;
 import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_TIMEOUT_DEFAULT;
@@ -252,7 +253,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     int threadCount = configuration.getInt(BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT, 32);
 
     ShardServerEventHandler eventHandler = new ShardServerEventHandler();
-
+    
     final ThriftBlurShardServer server = new ThriftBlurShardServer();
     server.setNodeName(nodeName);
     server.setServerTransport(tNonblockingServerSocket);
@@ -262,6 +263,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     server.setAcceptQueueSizePerThread(configuration.getInt(BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD,
4));
     server.setMaxReadBufferBytes(configuration.getLong(BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES,
Long.MAX_VALUE));
     server.setSelectorThreads(configuration.getInt(BLUR_SHARD_THRIFT_SELECTOR_THREADS, 2));
+    server.setMaxFrameSize(config.getInt(BLUR_THRIFT_MAX_FRAME_SIZE, 16384000));
 
     // This will shutdown the server when the correct path is set in zk
     BlurShutdown shutdown = new BlurShutdown() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
index 4cda152..48c0b27 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
@@ -63,7 +63,7 @@ import com.yammer.metrics.core.Gauge;
 import com.yammer.metrics.core.MetricName;
 
 public class ThriftServer {
-  
+
   private static final Log LOG = LogFactory.getLog(ThriftServer.class);
 
   private String _nodeName;
@@ -80,6 +80,15 @@ public class ThriftServer {
   private int _acceptQueueSizePerThread = 4;
   private long _maxReadBufferBytes = Long.MAX_VALUE;
   private int _selectorThreads = 2;
+  private int _maxFrameSize = 16384000;
+
+  public int getMaxFrameSize() {
+    return _maxFrameSize;
+  }
+
+  public void setMaxFrameSize(int maxFrameSize) {
+    _maxFrameSize = maxFrameSize;
+  }
 
   public TNonblockingServerSocket getServerTransport() {
     return _serverTransport;
@@ -204,10 +213,11 @@ public class ThriftServer {
   public void start() throws TTransportException {
     _executorService = Executors.newThreadPool("thrift-processors", _threadCount);
     Blur.Processor<Blur.Iface> processor = new Blur.Processor<Blur.Iface>(_iface);
+
     TThreadedSelectorServer.Args args = new TThreadedSelectorServer.Args(_serverTransport);
     args.processor(processor);
     args.executorService(_executorService);
-    args.transportFactory(new TFramedTransport.Factory());
+    args.transportFactory(new TFramedTransport.Factory(_maxFrameSize));
     args.protocolFactory(new TBinaryProtocol.Factory(true, true));
     args.selectorThreads = _selectorThreads;
     args.maxReadBufferBytes = _maxReadBufferBytes;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java b/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java
index 440eea1..ee9e3db 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/ClientPool.java
@@ -17,6 +17,10 @@ package org.apache.blur.thrift;
  * limitations under the License.
  */
 
+import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY;
+import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD;
+import static org.apache.blur.utils.BlurConstants.BLUR_THRIFT_MAX_FRAME_SIZE;
+
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
@@ -40,15 +44,15 @@ import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TFramedTransport;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TSocket;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException;
-import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD;
-import static org.apache.blur.utils.BlurConstants.BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY;
 import org.apache.blur.thrift.generated.Blur.Client;
 
 public class ClientPool {
 
+  
   private static final Log LOG = LogFactory.getLog(ClientPool.class);
   private static final Map<Connection, BlockingQueue<Client>> _connMap = new
ConcurrentHashMap<Connection, BlockingQueue<Client>>();
   private int _maxConnectionsPerHost = Integer.MAX_VALUE;
+  private static final int _maxFrameSize;
   private static AtomicBoolean _running = new AtomicBoolean(true);
   private static long _idleTimeBeforeClosingClient;
   private static long _clientPoolCleanFrequency;
@@ -61,6 +65,7 @@ public class ClientPool {
                       TimeUnit.SECONDS.toMillis(30));
       _clientPoolCleanFrequency = config.getLong(BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY,
               TimeUnit.SECONDS.toMillis(300)); 
+      _maxFrameSize = config.getInt(BLUR_THRIFT_MAX_FRAME_SIZE, 16384000);
     } catch (Exception e) {
         throw new RuntimeException(e);
     }
@@ -211,7 +216,7 @@ public class ClientPool {
     socket.connect(new InetSocketAddress(host, port), timeout);
     trans = new TSocket(socket);
 
-    TProtocol proto = new TBinaryProtocol(new TFramedTransport(trans));
+    TProtocol proto = new TBinaryProtocol(new TFramedTransport(trans,_maxFrameSize));
     return new WeightedClient(proto);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 525b4bd..094815a 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -120,6 +120,8 @@ public class BlurConstants {
   public static final String BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD = "blur.clientpool.client.close.threshold";
   public static final String BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY = "blur.clientpool.client.clean.frequency";
   
+  public static final String BLUR_THRIFT_MAX_FRAME_SIZE = "blur.thrift.max.frame.size";
+  
   public static final String BLUR_SHARD_FILTERED_SERVER_CLASS = "blur.shard.filtered.server.class";
   public static final String BLUR_CONTROLLER_FILTERED_SERVER_CLASS = "blur.controller.filtered.server.class";
   

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a348d49c/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 393bfb2..213c931 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -31,6 +31,9 @@ blur.query.max.record.fetch=1000
 # Setup metric reporter
 blur.metrics.reporters=
 
+# Thrift max frame size
+blur.thrift.max.frame.size=16384000
+
 ### Shard Server Configuration
 
 # The hostname for the shard, if blank the hostname is automatically detected


Mime
View raw message