zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject svn commit: r1307732 - in /zookeeper/bookkeeper/trunk: CHANGES.txt bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
Date Sat, 31 Mar 2012 07:31:54 GMT
Author: sijie
Date: Sat Mar 31 07:31:54 2012
New Revision: 1307732

URL: http://svn.apache.org/viewvc?rev=1307732&view=rev
Log:
BOOKKEEPER-198: replaying entries of deleted ledgers would exhaust ledger cache. (sijie)

Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1307732&r1=1307731&r2=1307732&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Sat Mar 31 07:31:54 2012
@@ -72,6 +72,8 @@ Trunk (unreleased changes)
 
         BOOKKEEPER-193: Ledger is garbage collected by mistake. (sijie, ivank via sijie)
 
+        BOOKKEEPER-198: replaying entries of deleted ledgers would exhaust ledger cache.
(sijie)
+
       hedwig-server/
       
         BOOKKEEPER-140: Hub server doesn't subscribe remote region correctly when a region
is down. (Sijie Gou via ivank)

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java?rev=1307732&r1=1307731&r2=1307732&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheImpl.java
Sat Mar 31 07:31:54 2012
@@ -153,6 +153,40 @@ public class LedgerCacheImpl implements 
         }
     }
 
+    /** 
+     * Grab ledger entry page whose first entry is <code>pageEntry</code>.
+     *
+     * If the page doesn't existed before, we allocate a memory page.
+     * Otherwise, we grab a clean page and read it from disk.
+     *
+     * @param ledger
+     *          Ledger Id
+     * @param pageEntry
+     *          Start entry of this entry page.
+     */
+    private LedgerEntryPage grabLedgerEntryPage(long ledger, long pageEntry) throws IOException
{
+        LedgerEntryPage lep = grabCleanPage(ledger, pageEntry);
+        try {
+            // should update page before we put it into table
+            // otherwise we would put an empty page in it
+            updatePage(lep);
+            synchronized(this) {
+                putIntoTable(pages, lep);
+            }   
+        } catch (IOException ie) {
+            // if we grab a clean page, but failed to update the page
+            // we are exhuasting the count of ledger entry pages.
+            // since this page will be never used, so we need to decrement
+            // page count of ledger cache.
+            lep.releasePage();
+            synchronized (this) {
+                --pageCount;
+            }
+            throw ie; 
+        }   
+        return lep;
+    }
+
     @Override
     public void putEntryOffset(long ledger, long entry, long offset) throws IOException {
         int offsetInPage = (int) (entry % entriesPerPage);
@@ -161,12 +195,7 @@ public class LedgerCacheImpl implements 
         long pageEntry = entry-offsetInPage;
         LedgerEntryPage lep = getLedgerEntryPage(ledger, pageEntry, false);
         if (lep == null) {
-            // find a free page
-            lep = grabCleanPage(ledger, pageEntry);
-            updatePage(lep);
-            synchronized(this) {
-                putIntoTable(pages, lep);
-            }
+            lep = grabLedgerEntryPage(ledger, pageEntry); 
         }
         if (lep != null) {
             lep.setOffset(offset, offsetInPage*8);
@@ -184,13 +213,7 @@ public class LedgerCacheImpl implements 
         LedgerEntryPage lep = getLedgerEntryPage(ledger, pageEntry, false);
         try {
             if (lep == null) {
-                lep = grabCleanPage(ledger, pageEntry);
-                // should update page before we put it into table
-                // otherwise we would put an empty page in it
-                updatePage(lep);
-                synchronized(this) {
-                    putIntoTable(pages, lep);
-                }
+                lep = grabLedgerEntryPage(ledger, pageEntry);
             }
             return lep.getOffset(offsetInPage*8);
         } finally {

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java?rev=1307732&r1=1307731&r2=1307732&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
Sat Mar 31 07:31:54 2012
@@ -24,6 +24,7 @@ package org.apache.bookkeeper.bookie;
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.bookkeeper.bookie.Bookie.NoLedgerException;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.meta.LedgerManager;
 import org.apache.bookkeeper.meta.LedgerManagerFactory;
@@ -124,4 +125,51 @@ public class LedgerCacheTest extends Tes
         }
     }
 
+    @Test
+    public void testPageEviction() throws Exception {
+        int numLedgers = 10;
+        byte[] masterKey = "blah".getBytes();
+        // limit page count
+        conf.setOpenFileLimit(999999).setPageLimit(3);
+        // create ledger cache
+        newLedgerCache();
+        try {
+            // create serveral ledgers
+            for (int i=1; i<=numLedgers; i++) {
+                ledgerCache.setMasterKey((long)i, masterKey);
+                ledgerCache.putEntryOffset(i, 0, i*8);
+                ledgerCache.putEntryOffset(i, 1, i*8);
+            }
+
+            // flush all first to clean previous dirty ledgers
+            ledgerCache.flushLedger(true);
+            // flush all 
+            ledgerCache.flushLedger(true);
+
+            // delete serveral ledgers
+            for (int i=1; i<=numLedgers/2; i++) {
+                ledgerCache.deleteLedger(i);
+            }
+
+            // bookie restarts
+            newLedgerCache();
+
+            // simulate replaying journals to add entries again
+            for (int i=1; i<=numLedgers; i++) {
+                try {
+                    ledgerCache.putEntryOffset(i, 1, i*8);
+                } catch (NoLedgerException nsle) {
+                    if (i<=numLedgers/2) {
+                        // it is ok
+                    } else {
+                        LOG.error("Error put entry offset : ", nsle);
+                        fail("Should not reach here.");
+                    }
+                }
+            }
+        } catch (Exception e) {
+            LOG.error("Got Exception.", e);
+            fail("Failed to add entry.");
+        }
+    }
 }



Mime
View raw message