bookkeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject bookkeeper git commit: BOOKKEEPER-968: Entry log flushes at configurable chunks
Date Tue, 31 Jan 2017 01:11:38 GMT
Repository: bookkeeper
Updated Branches:
  refs/heads/master ba5dadcb3 -> 26b09abb4


BOOKKEEPER-968: Entry log flushes at configurable chunks

With this patch one can configure interval (in bytes) for entry log to flush writes to the
disk.

Author: Andrey Yegorov <ayegorov@salesforce.com>

Reviewers: Sijie Guo <sijie@apache.org>

Closes #77 from dlg99/task/entry-log-flush


Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/26b09abb
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/26b09abb
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/26b09abb

Branch: refs/heads/master
Commit: 26b09abb4202362ca37d6944ce75eb2a3309dc3c
Parents: ba5dadc
Author: Andrey Yegorov <ayegorov@salesforce.com>
Authored: Mon Jan 30 17:11:29 2017 -0800
Committer: Sijie Guo <sijie@apache.org>
Committed: Mon Jan 30 17:11:29 2017 -0800

----------------------------------------------------------------------
 bookkeeper-server/conf/bk_server.conf           |  7 +++++
 .../apache/bookkeeper/bookie/EntryLogger.java   | 23 ++++++++++++++++
 .../bookkeeper/conf/ServerConfiguration.java    | 28 ++++++++++++++++++++
 3 files changed, 58 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b09abb/bookkeeper-server/conf/bk_server.conf
----------------------------------------------------------------------
diff --git a/bookkeeper-server/conf/bk_server.conf b/bookkeeper-server/conf/bk_server.conf
index 14c6068..e2a2be6 100644
--- a/bookkeeper-server/conf/bk_server.conf
+++ b/bookkeeper-server/conf/bk_server.conf
@@ -274,6 +274,13 @@ zkTimeout=10000
 # The interval is specified in seconds.
 #auditorPeriodicBookieCheckInterval=86400
 
+# Entry log flush interval in bytes.
+# Default is 0. 0 or less disables this feature and effectively flush
+# happens on log rotation.
+# Flushing in smaller chunks but more frequently reduces spikes in disk
+# I/O. Flushing too frequently may also affect performance negatively.
+#flushEntrylogBytes=0
+
 # How long to wait, in seconds, before starting auto recovery of a lost bookie
 #lostBookieRecoveryDelay=0
 

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b09abb/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index 08ad1be..3314903 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -171,6 +171,10 @@ public class EntryLogger {
     final static int MIN_SANE_ENTRY_SIZE = 8 + 8;
     final static long MB = 1024 * 1024;
 
+    private final long flushIntervalInBytes;
+    private final boolean doRegularFlushes;
+    private long bytesWrittenSinceLastFlush = 0;
+
     final ServerConfiguration conf;
     /**
      * Scan entries in a entry log file.
@@ -253,6 +257,9 @@ public class EntryLogger {
         this.leastUnflushedLogId = logId + 1;
         this.entryLoggerAllocator = new EntryLoggerAllocator(logId);
         this.conf = conf;
+        flushIntervalInBytes = conf.getFlushIntervalInBytes();
+        doRegularFlushes = flushIntervalInBytes > 0;
+
         initialize();
     }
 
@@ -737,6 +744,7 @@ public class EntryLogger {
     synchronized void flushCurrentLog() throws IOException {
         if (logChannel != null) {
             logChannel.flush(true);
+            bytesWrittenSinceLastFlush = 0;
             LOG.debug("Flush and sync current entry logger {}.", logChannel.getLogId());
         }
     }
@@ -752,6 +760,9 @@ public class EntryLogger {
         // Create new log if logSizeLimit reached or current disk is full
         boolean createNewLog = shouldCreateNewEntryLog.get();
         if (createNewLog || reachEntryLogLimit) {
+            if (doRegularFlushes) {
+                flushCurrentLog();
+            }
             createNewLog();
             // Reset the flag
             if (createNewLog) {
@@ -766,10 +777,22 @@ public class EntryLogger {
         long pos = logChannel.position();
         logChannel.write(entry);
         logChannel.registerWrittenEntry(ledger, entrySize);
+        
+        incrementBytesWrittenAndMaybeFlush(4L + entrySize);
 
         return (logChannel.getLogId() << 32L) | pos;
     }
 
+    private void incrementBytesWrittenAndMaybeFlush(long bytesWritten) throws IOException
{
+        if (!doRegularFlushes) {
+            return;
+        }
+        bytesWrittenSinceLastFlush += bytesWritten;
+        if (bytesWrittenSinceLastFlush > flushIntervalInBytes) {
+            flushCurrentLog();
+        }
+    }
+
     static long logIdForOffset(long offset) {
         return offset >> 32L;
     }

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b09abb/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 6bc118a..8ce6908 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -56,6 +56,7 @@ public class ServerConfiguration extends AbstractConfiguration {
     protected final static String GC_OVERREPLICATED_LEDGER_WAIT_TIME = "gcOverreplicatedLedgerWaitTime";
     // Sync Parameters
     protected final static String FLUSH_INTERVAL = "flushInterval";
+    protected final static String FLUSH_ENTRYLOG_INTERVAL_BYTES = "flushEntrylogBytes";
     // Bookie death watch interval
     protected final static String DEATH_WATCH_INTERVAL = "bookieDeathWatchInterval";
     // Ledger Cache Parameters
@@ -266,6 +267,33 @@ public class ServerConfiguration extends AbstractConfiguration {
     }
 
     /**
+     * Set entry log flush interval in bytes.
+     * 
+     * Default is 0. 0 or less disables this feature and effectively flush
+     * happens on log rotation.
+     *
+     * Flushing in smaller chunks but more frequently reduces spikes in disk
+     * I/O. Flushing too frequently may also affect performance negatively.
+     * 
+     * @return Entry log flush interval in bytes
+     */
+    public long getFlushIntervalInBytes() {
+        return this.getLong(FLUSH_ENTRYLOG_INTERVAL_BYTES, 0);
+    }
+
+    /**
+     * Set entry log flush interval in bytes
+     *
+     * @param flushInterval in bytes
+     * @return server configuration
+     */
+    public ServerConfiguration setFlushIntervalInBytes(long flushInterval) {
+        this.setProperty(FLUSH_ENTRYLOG_INTERVAL_BYTES, Long.toString(flushInterval));
+        return this;
+    }
+    
+    
+    /**
      * Get bookie death watch interval
      *
      * @return watch interval


Mime
View raw message