bookkeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject bookkeeper git commit: BOOKKEEPER-998: Increased the max entry size to 5MB
Date Wed, 22 Mar 2017 23:48:11 GMT
Repository: bookkeeper
Updated Branches:
  refs/heads/master 123eccd43 -> 28c264c86


BOOKKEEPER-998: Increased the max entry size to 5MB

Full disclosure, most of these changes were actually done by merlimat For this I am mostly
just putting them back into open source.

Author: Robert (Bobby) Evans <evans@yahoo-inc.com>

Reviewers: Enrico Olivelli <eolivelli@gmail.com>, Venkateswararao Jujjuri (JV) <None>

Closes #104 from revans2/BOOKKEEPER-998


Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/28c264c8
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/28c264c8
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/28c264c8

Branch: refs/heads/master
Commit: 28c264c8655f74e09ceee5ad499942cfb0dc65dc
Parents: 123eccd
Author: Robert (Bobby) Evans <evans@yahoo-inc.com>
Authored: Wed Mar 22 16:48:06 2017 -0700
Committer: Sijie Guo <sijie@apache.org>
Committed: Wed Mar 22 16:48:06 2017 -0700

----------------------------------------------------------------------
 .../apache/bookkeeper/bookie/EntryLogger.java   |  8 ++++--
 .../bookkeeper/conf/AbstractConfiguration.java  | 28 +++++++++++++++++++-
 .../bookkeeper/conf/ClientConfiguration.java    | 10 +++++++
 .../bookkeeper/conf/ServerConfiguration.java    | 10 +++++++
 .../bookkeeper/proto/BookieNettyServer.java     |  5 ++--
 .../proto/PerChannelBookieClient.java           |  5 ++--
 6 files changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/28c264c8/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index 3314903..69b7d24 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -174,6 +174,7 @@ public class EntryLogger {
     private final long flushIntervalInBytes;
     private final boolean doRegularFlushes;
     private long bytesWrittenSinceLastFlush = 0;
+    private final int maxSaneEntrySize;
 
     final ServerConfiguration conf;
     /**
@@ -226,6 +227,9 @@ public class EntryLogger {
     public EntryLogger(ServerConfiguration conf,
             LedgerDirsManager ledgerDirsManager, EntryLogListener listener)
                     throws IOException {
+        //We reserve 500 bytes as overhead for the protocol.  This is not 100% accurate
+        // but the protocol varies so an exact value is difficult to determine
+        this.maxSaneEntrySize = conf.getNettyMaxFrameSizeBytes() - 500;
         this.ledgerDirsManager = ledgerDirsManager;
         if (listener != null) {
             addListener(listener);
@@ -826,8 +830,8 @@ public class EntryLogger {
         sizeBuff.flip();
         int entrySize = sizeBuff.getInt();
         // entrySize does not include the ledgerId
-        if (entrySize > MB) {
-            LOG.error("Sanity check failed for entry size of " + entrySize + " at location
" + pos + " in " + entryLogId);
+        if (entrySize > maxSaneEntrySize) {
+            LOG.warn("Sanity check failed for entry size of " + entrySize + " at location
" + pos + " in " + entryLogId);
 
         }
         if (entrySize < MIN_SANE_ENTRY_SIZE) {

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/28c264c8/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
index 7df41fc..1497c7a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
@@ -18,7 +18,6 @@
 package org.apache.bookkeeper.conf;
 
 import java.net.URL;
-import static org.apache.bookkeeper.conf.ClientConfiguration.CLIENT_AUTH_PROVIDER_FACTORY_CLASS;
 
 import org.apache.commons.configuration.CompositeConfiguration;
 import org.apache.commons.configuration.Configuration;
@@ -70,6 +69,10 @@ public abstract class AbstractConfiguration extends CompositeConfiguration
{
     // Client auth provider factory class name. It must be configured on Bookies to for the
Auditor
     protected final static String CLIENT_AUTH_PROVIDER_FACTORY_CLASS = "clientAuthProviderFactoryClass";
 
+    //Netty configuration
+    protected final static String NETTY_MAX_FRAME_SIZE = "nettyMaxFrameSizeBytes";
+    protected final static int DEFAULT_NETTY_MAX_FRAME_SIZE = 5 * 1024 * 1024; // 5MB
+
     protected AbstractConfiguration() {
         super();
         if (READ_SYSTEM_PROPERTIES) {
@@ -284,4 +287,27 @@ public abstract class AbstractConfiguration extends CompositeConfiguration
{
     public String getClientAuthProviderFactoryClass() {
         return getString(CLIENT_AUTH_PROVIDER_FACTORY_CLASS, null);
     }
+
+    /**
+     * Get the maximum netty frame size in bytes.  Any message received larger
+     * that this will be rejected.
+     *
+     * @return the maximum netty frame size in bytes.
+     */
+    public int getNettyMaxFrameSizeBytes() {
+        return getInt(NETTY_MAX_FRAME_SIZE, DEFAULT_NETTY_MAX_FRAME_SIZE);
+    }
+
+    /**
+     * Set the max number of bytes a single message can be that is read by the bookie.
+     * Any message larger than that size will be rejected.
+     *
+     * @param maxSize
+     *          the max size in bytes
+     * @return server configuration
+     */
+    public AbstractConfiguration setNettyMaxFrameSizeBytes(int maxSize) {
+        setProperty(NETTY_MAX_FRAME_SIZE, String.valueOf(maxSize));
+        return this;
+    }
 }

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/28c264c8/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
index fa42dc9..b73d2e2 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
@@ -57,6 +57,7 @@ public class ClientConfiguration extends AbstractConfiguration {
     protected final static String CLIENT_WRITEBUFFER_HIGH_WATER_MARK = "clientWriteBufferHighWaterMark";
     protected final static String CLIENT_CONNECT_TIMEOUT_MILLIS = "clientConnectTimeoutMillis";
     protected final static String NUM_CHANNELS_PER_BOOKIE = "numChannelsPerBookie";
+
     // Read Parameters
     protected final static String READ_TIMEOUT = "readTimeout";
     protected final static String SPECULATIVE_READ_TIMEOUT = "speculativeReadTimeout";
@@ -931,6 +932,15 @@ public class ClientConfiguration extends AbstractConfiguration {
     }
 
     /**
+     * {@inheritDoc}
+     */
+    @Override
+    public ClientConfiguration setNettyMaxFrameSizeBytes(int maxSize) {
+        super.setNettyMaxFrameSizeBytes(maxSize);
+        return this;
+    }
+
+    /**
      * Set the client role
      *
      * @param role defines how the client will act

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/28c264c8/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index b590e38..986f9ed 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -90,6 +90,7 @@ public class ServerConfiguration extends AbstractConfiguration {
     protected final static String ALLOW_STORAGE_EXPANSION = "allowStorageExpansion";
     // NIO Parameters
     protected final static String SERVER_TCP_NODELAY = "serverTcpNoDelay";
+
     // Zookeeper Parameters
     protected final static String ZK_TIMEOUT = "zkTimeout";
     protected final static String ZK_SERVERS = "zkServers";
@@ -1772,4 +1773,13 @@ public class ServerConfiguration extends AbstractConfiguration {
     public String getBookieAuthProviderFactoryClass() {
         return getString(BOOKIE_AUTH_PROVIDER_FACTORY_CLASS, null);
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public ServerConfiguration setNettyMaxFrameSizeBytes(int maxSize) {
+        super.setNettyMaxFrameSizeBytes(maxSize);
+        return this;
+    }
 }

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/28c264c8/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
index 924c887..2c6dd3a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
@@ -59,7 +59,7 @@ class BookieNettyServer {
 
     private final static Logger LOG = LoggerFactory.getLogger(BookieNettyServer.class);
 
-    final static int maxMessageSize = 0xfffff;
+    final int maxFrameSize;
     final ServerConfiguration conf;
     final List<ChannelManager> channels = new ArrayList<>();
     final RequestProcessor requestProcessor;
@@ -74,6 +74,7 @@ class BookieNettyServer {
 
     BookieNettyServer(ServerConfiguration conf, RequestProcessor processor)
         throws IOException, KeeperException, InterruptedException, BookieException {
+        this.maxFrameSize = conf.getNettyMaxFrameSizeBytes();
         this.conf = conf;
         this.requestProcessor = processor;
 
@@ -207,7 +208,7 @@ class BookieNettyServer {
             BookieSideConnectionPeerContextHandler contextHandler = new BookieSideConnectionPeerContextHandler();
             ChannelPipeline pipeline = Channels.pipeline();
             pipeline.addLast("lengthbaseddecoder",
-                new LengthFieldBasedFrameDecoder(maxMessageSize, 0, 4, 0, 4));
+                new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4));
             pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
 
             pipeline.addLast("bookieProtoDecoder", requestDecoder);

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/28c264c8/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index 3fb73e4..9d32ff8 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
@@ -118,7 +118,6 @@ public class PerChannelBookieClient extends SimpleChannelHandler implements
Chan
                         BKException.Code.DuplicateEntryIdException,
                         BKException.Code.WriteOnReadOnlyBookieException));
 
-    public static final int MAX_FRAME_LENGTH = 2 * 1024 * 1024; // 2M
     public static final AtomicLong txnIdGenerator = new AtomicLong(0);
 
     final BookieSocketAddress addr;
@@ -127,6 +126,7 @@ public class PerChannelBookieClient extends SimpleChannelHandler implements
Chan
     final HashedWheelTimer requestTimer;
     final int addEntryTimeout;
     final int readEntryTimeout;
+    final int maxFrameSize;
 
     private final ConcurrentHashMap<CompletionKey, CompletionValue> completionObjects
= new ConcurrentHashMap<CompletionKey, CompletionValue>();
 
@@ -181,6 +181,7 @@ public class PerChannelBookieClient extends SimpleChannelHandler implements
Chan
                                   ClientAuthProvider.Factory authProviderFactory,
                                   ExtensionRegistry extRegistry,
                                   PerChannelBookieClientPool pcbcPool) {
+        this.maxFrameSize = conf.getNettyMaxFrameSizeBytes();
         this.conf = conf;
         this.addr = addr;
         this.executor = executor;
@@ -893,7 +894,7 @@ public class PerChannelBookieClient extends SimpleChannelHandler implements
Chan
     public ChannelPipeline getPipeline() throws Exception {
         ChannelPipeline pipeline = Channels.pipeline();
 
-        pipeline.addLast("lengthbasedframedecoder", new LengthFieldBasedFrameDecoder(MAX_FRAME_LENGTH,
0, 4, 0, 4));
+        pipeline.addLast("lengthbasedframedecoder", new LengthFieldBasedFrameDecoder(maxFrameSize,
0, 4, 0, 4));
         pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
         pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.RequestEncoder(extRegistry));
         pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.ResponseDecoder(extRegistry));


Mime
View raw message