bookkeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject bookkeeper git commit: BOOKKEEPER-949: Allow entryLog creation for compaction
Date Mon, 30 Jan 2017 22:25:06 GMT
Repository: bookkeeper
Updated Branches:
  refs/heads/master ca7e5fd80 -> ba5dadcb3


BOOKKEEPER-949: Allow entryLog creation for compaction

Allow entryLog creation even when bookie
is in RO mode for compaction

Signed-off-by: Venkateswararao Jujjuri (JV) <vjujjurisalesforce.com>
Reviewed-by: Andrey Yegorov <ayegorovsalesforce.com>
Reviewed-by: Charan Reddy Guttapalem <cguttapalemsalesforce.com>

Author: JV <vjujjuri@salesforce.com>

Reviewers: Sijie Guo <sijie@apache.org>

Closes #99 from reddycharan/entrylogcreationforcompaction


Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/ba5dadcb
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/ba5dadcb
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/ba5dadcb

Branch: refs/heads/master
Commit: ba5dadcb3e1fa942f2f1a17178fd368c4bd0f0c4
Parents: ca7e5fd
Author: JV <vjujjuri@salesforce.com>
Authored: Mon Jan 30 14:25:01 2017 -0800
Committer: Sijie Guo <sijie@apache.org>
Committed: Mon Jan 30 14:25:01 2017 -0800

----------------------------------------------------------------------
 .../apache/bookkeeper/bookie/BookieShell.java   |  8 +++-
 .../apache/bookkeeper/bookie/EntryLogger.java   |  2 +-
 .../bookie/GarbageCollectorThread.java          |  2 +-
 .../bookkeeper/bookie/LedgerDirsManager.java    | 44 ++++++++++++++++++++
 .../bookkeeper/bookie/CreateNewLogTest.java     | 32 +++++++++++++-
 .../bookie/TestLedgerDirsManager.java           | 30 +++++++++++++
 6 files changed, 113 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/ba5dadcb/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 1092001..b4030f1 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -23,6 +23,7 @@ import static com.google.common.base.Charsets.UTF_8;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -1818,8 +1819,11 @@ public class BookieShell implements Tool {
         Collections.sort(completeFilesList, new FilesTimeComparator());
         return completeFilesList;
     }
-    
-    private static class FilesTimeComparator implements Comparator<File> {
+
+    private static class FilesTimeComparator implements Comparator<File>, Serializable
{
+
+        private static final long serialVersionUID = 1L;
+
         @Override
         public int compare(File file1, File file2) {
             Path file1Path = Paths.get(file1.getAbsolutePath());

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/ba5dadcb/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index ecd0e0a..08ad1be 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -549,7 +549,7 @@ public class EntryLogger {
          * Allocate a new log file.
          */
         BufferedLogChannel allocateNewLog() throws IOException {
-            List<File> list = ledgerDirsManager.getWritableLedgerDirs();
+            List<File> list = ledgerDirsManager.getWritableLedgerDirsForNewLog();
             Collections.shuffle(list);
             // It would better not to overwrite existing entry log files
             File newLogFile = null;

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/ba5dadcb/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index 36b7349..aa42475 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -504,7 +504,7 @@ public class GarbageCollectorThread extends BookieThread {
         // closed and corrupted.
         if (!compacting.compareAndSet(false, true)) {
             // set compacting flag failed, means compacting is true now
-            // indicates another thread wants to interrupt gc thread to exit
+            // indicates that compaction is in progress for this EntryLogId.
             return;
         }
 

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/ba5dadcb/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
index 7c95b85..52836f7 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
@@ -60,6 +60,8 @@ public class LedgerDirsManager {
     private final Random rand = new Random();
     private final ConcurrentMap<File, Float> diskUsages =
             new ConcurrentHashMap<File, Float>();
+    private final long entryLogSize;
+    private boolean forceGCAllowWhenNoSpace;
 
     public LedgerDirsManager(ServerConfiguration conf, File[] dirs) {
         this(conf, dirs, NullStatsLogger.INSTANCE);
@@ -78,6 +80,8 @@ public class LedgerDirsManager {
         this.listeners = new ArrayList<LedgerDirsListener>();
         this.diskChecker = diskChecker;
         this.monitor = new LedgerDirsMonitor(conf.getDiskCheckInterval());
+        this.forceGCAllowWhenNoSpace = conf.getIsForceGCAllowWhenNoSpace();
+        this.entryLogSize = conf.getEntryLogSizeLimit();
         for (File dir : dirs) {
             diskUsages.put(dir, 0f);
             String statName = "dir_" + dir.getPath().replace('/', '_') + "_usage";
@@ -129,6 +133,46 @@ public class LedgerDirsManager {
         return writableLedgerDirectories;
     }
 
+    public List<File> getWritableLedgerDirsForNewLog()
+        throws NoWritableLedgerDirException {
+
+        if (!writableLedgerDirectories.isEmpty()) {
+            return writableLedgerDirectories;
+        }
+
+        // If Force GC is not allowed under no space
+        if (!forceGCAllowWhenNoSpace) {
+            String errMsg = "All ledger directories are non writable and force GC is not
enabled.";
+            NoWritableLedgerDirException e = new NoWritableLedgerDirException(errMsg);
+            LOG.error(errMsg, e);
+            throw e;
+        }
+
+        // We don't have writable Ledger Dirs.
+        // That means we must have turned readonly but the compaction
+        // must have started running and it needs to allocate
+        // a new log file to move forward with the compaction.
+        List<File> fullLedgerDirsToAccomodateNewEntryLog = new ArrayList<File>();
+        for (File dir: this.ledgerDirectories) {
+            // Pick dirs which can accommodate little more than an entry log.
+            if (dir.getUsableSpace() > (this.entryLogSize * 1.2) ) {
+                fullLedgerDirsToAccomodateNewEntryLog.add(dir);
+            }
+        }
+
+        if (!fullLedgerDirsToAccomodateNewEntryLog.isEmpty()) {
+            LOG.info("No writable ledger dirs. Trying to go beyond to accomodate compaction."
+                    + "Dirs that can accomodate new entryLog are: {}", fullLedgerDirsToAccomodateNewEntryLog);
+            return fullLedgerDirsToAccomodateNewEntryLog;
+        }
+
+        // We will reach here when we have no option of creating a new log file for compaction
+        String errMsg = "All ledger directories are non writable and no reserved space left
for creating entry log file.";
+        NoWritableLedgerDirException e = new NoWritableLedgerDirException(errMsg);
+        LOG.error(errMsg, e);
+        throw e;
+    }
+
     /**
      * @return full-filled ledger dirs.
      */

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/ba5dadcb/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CreateNewLogTest.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CreateNewLogTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CreateNewLogTest.java
index e2d2e63..42167a3 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CreateNewLogTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CreateNewLogTest.java
@@ -19,9 +19,9 @@
 package org.apache.bookkeeper.bookie;
 
 import java.io.File;
+import java.util.List;
 
 import org.junit.Assert;
-
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.conf.TestBKConfiguration;
 import org.junit.After;
@@ -99,4 +99,34 @@ public class CreateNewLogTest {
         Assert.assertTrue("Wrong log id", el.getCurrentLogId() > 1);
     }
 
+    @Test(timeout=60000)
+    public void testCreateNewLogWithNoWritableLedgerDirs() throws Exception {
+        ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
+
+        // Creating a new configuration with a number of ledger directories.
+        conf.setLedgerDirNames(ledgerDirs);
+        conf.setIsForceGCAllowWhenNoSpace(true);
+        LedgerDirsManager ledgerDirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs());
+        EntryLogger el = new EntryLogger(conf, ledgerDirsManager);
+
+        // Extracted from createNewLog()
+        String logFileName = Long.toHexString(1) + ".log";
+        File dir = ledgerDirsManager.pickRandomWritableDir();
+        LOG.info("Picked this directory: " + dir);
+        File newLogFile = new File(dir, logFileName);
+        newLogFile.createNewFile();
+
+        // Now let us move all dirs to filled dirs
+        List<File> wDirs = ledgerDirsManager.getWritableLedgerDirs();
+        for (File tdir: wDirs) {
+            ledgerDirsManager.addToFilledDirs(tdir);
+        }
+
+        // Calls createNewLog, and with the number of directories we
+        // are using, if it picks one at random it will fail.
+        el.createNewLog();
+        LOG.info("This is the current log id: " + el.getCurrentLogId());
+        Assert.assertTrue("Wrong log id", el.getCurrentLogId() > 1);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/ba5dadcb/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
index 4d8d547..b72cfc7 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
@@ -72,6 +72,7 @@ public class TestLedgerDirsManager {
         ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
         conf.setLedgerDirNames(new String[] { tmpDir.toString() });
         conf.setDiskCheckInterval(diskCheckInterval);
+        conf.setIsForceGCAllowWhenNoSpace(true);
 
         mockDiskChecker = new MockDiskChecker(threshold, warnThreshold);
         dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), NullStatsLogger.INSTANCE,
mockDiskChecker);
@@ -88,6 +89,16 @@ public class TestLedgerDirsManager {
     }
 
     @Test(timeout=60000)
+    public void testGetWritableDir() throws Exception {
+        try {
+            List<File> writeDirs = dirsManager.getWritableLedgerDirs();
+            assertTrue("Must have a writable ledgerDir", writeDirs.size() > 0);
+        } catch (NoWritableLedgerDirException nwlde) {
+            fail("We should have a writeble ledgerDir");
+        }
+    }
+
+    @Test(timeout=60000)
     public void testPickWritableDirExclusive() throws Exception {
         try {
             dirsManager.pickRandomWritableDir(curDir);
@@ -112,6 +123,25 @@ public class TestLedgerDirsManager {
     }
 
     @Test(timeout=60000)
+    public void testGetWritableDirForLog() throws Exception {
+        List<File> writeDirs;
+        try {
+            dirsManager.addToFilledDirs(curDir);
+            writeDirs = dirsManager.getWritableLedgerDirs();
+            fail("Should not reach here due to there is no writable ledger dir.");
+        } catch (NoWritableLedgerDirException nwlde) {
+            // expected to fail with no writable ledger dir
+            // Now make sure we can get one for log
+            try {
+                writeDirs = dirsManager.getWritableLedgerDirsForNewLog();
+                assertTrue("Must have a writable ledgerDir", writeDirs.size() > 0);
+            } catch (NoWritableLedgerDirException e) {
+                fail("We should have a writeble ledgerDir");
+            }
+        }
+    }
+
+    @Test(timeout=60000)
     public void testLedgerDirsMonitorDuringTransition() throws Exception {
 
         MockLedgerDirsListener mockLedgerDirsListener = new MockLedgerDirsListener();


Mime
View raw message