zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iv...@apache.org
Subject svn commit: r1245411 - in /zookeeper/bookkeeper/trunk: CHANGES.txt bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieJournalRollingTest.java
Date Fri, 17 Feb 2012 11:50:24 GMT
Author: ivank
Date: Fri Feb 17 11:50:23 2012
New Revision: 1245411

URL: http://svn.apache.org/viewvc?rev=1245411&view=rev
Log:
BOOKKEEPER-174: Bookie can't start when replaying entries whose ledger were deleted and garbage
collected. (sijie via ivank)

Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieJournalRollingTest.java

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1245411&r1=1245410&r2=1245411&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Fri Feb 17 11:50:23 2012
@@ -42,6 +42,8 @@ Trunk (unreleased changes)
 
         BOOKKEEPER-169: bookie hangs on reading header when encountering partial header index
file (sijie via ivank)
 
+        BOOKKEEPER-174: Bookie can't start when replaying entries whose ledger were deleted
and garbage collected. (sijie via ivank)
+
       hedwig-server/
       
         BOOKKEEPER-140: Hub server doesn't subscribe remote region correctly when a region
is down. (Sijie Gou via ivank)

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java?rev=1245411&r1=1245410&r2=1245411&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
Fri Feb 17 11:50:23 2012
@@ -340,30 +340,34 @@ public class Bookie extends Thread {
                 }
                 recBuff.flip();
                 long ledgerId = recBuff.getLong();
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("Relay journal - ledger id : " + ledgerId);
-                }
-                LedgerDescriptor handle = getHandle(ledgerId);
                 long entryId = recBuff.getLong();
-                if (entryId == METAENTRY_ID_LEDGER_KEY) {
-                    if (recLog.getFormatVersion() >= 3) {
-                        int masterKeyLen = recBuff.getInt();
-                        byte[] masterKey = new byte[masterKeyLen];
-                        recBuff.get(masterKey);
-                        handle.checkAccess(masterKey);
-                        putHandle(handle);
+                LedgerDescriptor handle;
+                try {
+                    handle = getHandle(ledgerId);
+                    LOG.debug("Relay journal - ledger id : {}, entry id : {}.", ledgerId,
entryId);
+                    if (entryId == METAENTRY_ID_LEDGER_KEY) {
+                        if (recLog.getFormatVersion() >= 3) {
+                            int masterKeyLen = recBuff.getInt();
+                            byte[] masterKey = new byte[masterKeyLen];
+                            recBuff.get(masterKey);
+                            handle.checkAccess(masterKey);
+                            putHandle(handle);
+                        } else {
+                            throw new IOException("Invalid journal. Contains journalKey "
+                                    + " but layout version (" + recLog.getFormatVersion()
+                                    + ") is too old to hold this");
+                        }
                     } else {
-                        throw new IOException("Invalid journal. Contains journalKey "
-                                + " but layout version (" + recLog.getFormatVersion()
-                                + ") is too old to hold this");
-                    }
-                } else {
-                    try {
-                        recBuff.rewind();
-                        handle.addEntry(recBuff);
-                    } finally {
-                        putHandle(handle);
+                        try {
+                            recBuff.rewind();
+                            handle.addEntry(recBuff);
+                        } finally {
+                            putHandle(handle);
+                        }
                     }
+                } catch (NoLedgerException nsle) {
+                    LOG.debug("Skip replaying entries of ledger {} since it was deleted.",
ledgerId);
+                    continue;
                 }
             }
             recLog.close();

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieJournalRollingTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieJournalRollingTest.java?rev=1245411&r1=1245410&r2=1245411&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieJournalRollingTest.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieJournalRollingTest.java
Fri Feb 17 11:50:23 2012
@@ -79,7 +79,12 @@ public class BookieJournalRollingTest ex
             lhs[i] = bkc.createLedger(digestType, "".getBytes());
             ledgerIds[i] = lhs[i].getId();
         }
+        writeLedgerEntries(lhs, msgSize, numMsgs);
+        // Return the ledger handles to the inserted ledgers and entries
+        return lhs;
+    }
 
+    private void writeLedgerEntries(LedgerHandle[] lhs, int msgSize, int numMsgs) throws
Exception {
         // Create a dummy message string to write as ledger entries
         StringBuilder msgSB = new StringBuilder();
         for (int i = 0; i < msgSize; i++) {
@@ -89,16 +94,13 @@ public class BookieJournalRollingTest ex
 
         // Write all of the entries for all of the ledgers
         for (int i = 0; i < numMsgs; i++) {
-            for (int j = 0; j < numLedgers; j++) {
+            for (int j = 0; j < lhs.length; j++) {
                 StringBuilder sb = new StringBuilder();
-                sb.append(ledgerIds[j]).append('-').append(i).append('-')
+                sb.append(lhs[j].getId()).append('-').append(i).append('-')
                   .append(msg);
                 lhs[j].addEntry(sb.toString().getBytes());
             }
         }
-
-        // Return the ledger handles to the inserted ledgers and entries
-        return lhs;
     }
 
     private void validLedgerEntries(long[] ledgerIds, int msgSize, int numMsgs) throws Exception
{
@@ -216,4 +218,41 @@ public class BookieJournalRollingTest ex
         validLedgerEntries(ledgerIds, 1024, 1024);
     }
 
+    /**
+     * This test writes enough ledger entries to roll over the journals
+     * without sync up
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testReplayDeletedLedgerJournalEntries() throws Exception {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Testing replaying journal entries whose ledger has been removed.");
+        }
+
+        // Write entries
+        LedgerHandle[] lhs = writeLedgerEntries(1, 1024, 10);
+        // Wait until all entries are flushed and last mark rolls
+        Thread.sleep(3 * baseConf.getFlushInterval());
+
+        // restart bookies with flush interval set to a large value
+        ServerConfiguration newConf = new ServerConfiguration();
+        newConf.setFlushInterval(999999999);
+        // restart bookies
+        restartBookies(newConf);
+
+        // Write entries again to let them existed in journal
+        writeLedgerEntries(lhs, 1024, 10);
+
+        // delete them
+        for (LedgerHandle lh : lhs) {
+            bkc.deleteLedger(lh.getId());
+        }
+        // wait for gc
+        Thread.sleep(2 * newConf.getGcWaitTime());
+
+        // restart bookies again to trigger replaying journal
+        restartBookies(newConf);
+    }
+
 }



Mime
View raw message