zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iv...@apache.org
Subject svn commit: r1242404 - in /zookeeper/bookkeeper/trunk: ./ bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ bookkeeper-...
Date Thu, 09 Feb 2012 17:14:30 GMT
Author: ivank
Date: Thu Feb  9 17:14:29 2012
New Revision: 1242404

URL: http://svn.apache.org/viewvc?rev=1242404&view=rev
Log:
BOOKKEEPER-137: Do not create Ledger index files until absolutely necessary. (ivank)

Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Thu Feb  9 17:14:29 2012
@@ -49,6 +49,8 @@ Trunk (unreleased changes)
 
         BOOKKEEPER-165: Add versioning support for journal files (ivank)
 
+        BOOKKEEPER-137: Do not create Ledger index files until absolutely necessary. (ivank)
+
       hedwig-server/
 
         BOOKKEEPER-77: Add a console client for hedwig (Sijie Guo via ivank)

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
Thu Feb  9 17:14:29 2012
@@ -87,9 +87,12 @@ public class Bookie extends Thread {
      * when you make a change to the format of any of the files in 
      * this directory or to the general layout of the directory.
      */
-    static final int CURRENT_DIRECTORY_LAYOUT_VERSION = 1;
+    static final int MIN_COMPAT_DIRECTORY_LAYOUT_VERSION = 1;
+    static final int CURRENT_DIRECTORY_LAYOUT_VERSION = 2;
     static final String VERSION_FILENAME = "VERSION";
-    
+
+    static final long METAENTRY_ID_LEDGER_KEY = -0x1000;
+
     // ZK registration path for this bookie
     static final String BOOKIE_REGISTRATION_PATH = "/ledgers/available/";
 
@@ -250,7 +253,7 @@ public class Bookie extends Thread {
     }
 
     public Bookie(ServerConfiguration conf) 
-            throws IOException, KeeperException, InterruptedException {
+            throws IOException, KeeperException, InterruptedException, BookieException {
         super("Bookie-" + conf.getBookiePort());
         this.conf = conf;
         this.journalDirectory = conf.getJournalDir();
@@ -336,12 +339,28 @@ public class Bookie extends Thread {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("Relay journal - ledger id : " + ledgerId);
                 }
-                LedgerDescriptor handle = getHandle(ledgerId, false);
-                try {
-                    recBuff.rewind();
-                    handle.addEntry(recBuff);
-                } finally {
-                    putHandle(handle);
+                LedgerDescriptor handle = getHandle(ledgerId);
+                long entryId = recBuff.getLong();
+                if (entryId == METAENTRY_ID_LEDGER_KEY) {
+                    if (recLog.getFormatVersion() >= 3) {
+                        int masterKeyLen = recBuff.getInt();
+                        byte[] masterKey = new byte[masterKeyLen];
+                        recBuff.get(masterKey);
+                        
+                        handle.checkAccess(masterKey);
+                        putHandle(handle);
+                    } else {
+                        throw new IOException("Invalid journal. Contains journalKey "
+                                + " but layout version (" + recLog.getFormatVersion() 
+                                + ") is too old to hold this");
+                    }
+                } else {
+                    try {
+                        recBuff.rewind();
+                        handle.addEntry(recBuff);
+                    } finally {
+                        putHandle(handle);
+                    }
                 }
             }
             recLog.close();
@@ -540,7 +559,7 @@ public class Bookie extends Thread {
      * @throws IOException if layout version if is outside usable range
      *               or if there is a problem reading the version file
      */
-    private void checkDirectoryLayoutVersion(File dir) 
+    private void checkDirectoryLayoutVersion(File dir)
             throws IOException {
         if (!dir.isDirectory()) {
             throw new IOException("Directory("+dir+") isn't a directory");
@@ -566,8 +585,10 @@ public class Bookie extends Thread {
         try {
             String layoutVersionStr = br.readLine();
             int layoutVersion = Integer.parseInt(layoutVersionStr);
-            if (layoutVersion != CURRENT_DIRECTORY_LAYOUT_VERSION) {
-                String errmsg = "Directory has an invalid version, expected " 
+            if (layoutVersion < MIN_COMPAT_DIRECTORY_LAYOUT_VERSION
+                || layoutVersion > CURRENT_DIRECTORY_LAYOUT_VERSION) {
+                String errmsg = "Directory has an invalid version, expected between "
+                    + MIN_COMPAT_DIRECTORY_LAYOUT_VERSION + " and "
                     + CURRENT_DIRECTORY_LAYOUT_VERSION + ", found " + layoutVersion;
                 LOG.error(errmsg);
                 throw new IOException(errmsg);
@@ -620,73 +641,38 @@ public class Bookie extends Thread {
         }
     }
 
-    private LedgerDescriptor getHandle(long ledgerId, boolean readonly, byte[] masterKey)
throws IOException {
+    private LedgerDescriptor getHandle(long ledgerId, boolean readonly, byte[] masterKey)

+            throws IOException, BookieException {
         LedgerDescriptor handle = null;
         synchronized (ledgers) {
             handle = ledgers.get(ledgerId);
             if (handle == null) {
-                FileInfo fi = null;
-                try {
-                    // get file info will throw NoLedgerException
-                    fi = ledgerCache.getFileInfo(ledgerId, !readonly);
-
-                    // if an existed ledger index file, we can get its master key
-                    // if an new created ledger index file, we will get a null master key
-                    byte[] existingMasterKey = fi.readMasterKey();
-                    ByteBuffer masterKeyToSet = ByteBuffer.wrap(masterKey);
-                    if (existingMasterKey == null) {
-                        // no master key set before
-                        fi.writeMasterKey(masterKey);
-                    } else if (!masterKeyToSet.equals(ByteBuffer.wrap(existingMasterKey)))
{
-                        throw new IOException("Wrong master key for ledger " + ledgerId);
-                    }
-                    handle = createHandle(ledgerId, readonly);
-                    ledgers.put(ledgerId, handle);
-                    handle.setMasterKey(masterKeyToSet);
-                } finally {
-                    if (fi != null) {
-                        fi.release();
-                    }
+                if (readonly) {
+                    throw new NoLedgerException(ledgerId);
                 }
+                handle = createHandle(ledgerId);
+                ledgers.put(ledgerId, handle);
             }
+            handle.checkAccess(masterKey);
             handle.incRef();
         }
         return handle;
     }
 
-    private LedgerDescriptor getHandle(long ledgerId, boolean readonly) throws IOException
{
+    private LedgerDescriptor getHandle(long ledgerId) throws IOException {
         LedgerDescriptor handle = null;
         synchronized (ledgers) {
             handle = ledgers.get(ledgerId);
             if (handle == null) {
-                FileInfo fi = null;
-                try {
-                    // get file info will throw NoLedgerException
-                    fi = ledgerCache.getFileInfo(ledgerId, !readonly);
-
-                    // if an existed ledger index file, we can get its master key
-                    // if an new created ledger index file, we will get a null master key
-                    byte[] existingMasterKey = fi.readMasterKey();
-                    if (existingMasterKey == null) {
-                        throw new IOException("Weird! No master key found in ledger " + ledgerId);
-                    }
-
-                    handle = createHandle(ledgerId, readonly);
-                    ledgers.put(ledgerId, handle);
-                    handle.setMasterKey(ByteBuffer.wrap(existingMasterKey));
-                } finally {
-                    if (fi != null) {
-                        fi.release();
-                    }
-                }
+                handle = createHandle(ledgerId);
+                ledgers.put(ledgerId, handle);
             }
             handle.incRef();
         }
         return handle;
     }
 
-
-    private LedgerDescriptor createHandle(long ledgerId, boolean readOnly) throws IOException
{
+    private LedgerDescriptor createHandle(long ledgerId) throws IOException {
         return new LedgerDescriptor(ledgerId, entryLogger, ledgerCache);
     }
 
@@ -920,13 +906,28 @@ public class Bookie extends Thread {
     private LedgerDescriptor getLedgerForEntry(ByteBuffer entry, byte[] masterKey) 
             throws IOException, BookieException {
         long ledgerId = entry.getLong();
-        LedgerDescriptor handle = getHandle(ledgerId, false, masterKey);
-
-        if(!handle.cmpMasterKey(ByteBuffer.wrap(masterKey))) {
-            putHandle(handle);
-            throw BookieException.create(BookieException.Code.UnauthorizedAccessException);
+        LedgerDescriptor l = getHandle(ledgerId, false, masterKey);
+        if (!l.isMasterKeyPersisted()) {
+            // new handle, we should add the key to journal ensure we can rebuild
+            ByteBuffer bb = ByteBuffer.allocate(8 + 8 + 4 + masterKey.length);
+            bb.putLong(ledgerId);
+            bb.putLong(METAENTRY_ID_LEDGER_KEY);
+            bb.putInt(masterKey.length);
+            bb.put(masterKey);
+            bb.flip();
+
+            queue.add(new QueueEntry(bb,
+                                     ledgerId, METAENTRY_ID_LEDGER_KEY,
+                                     new WriteCallback() {
+                                         public void writeComplete(int rc, long ledgerId,

+                                                 long entryId, InetSocketAddress addr,
+                                                 Object ctx) {
+                                             // do nothing
+                                         }
+                                     }, null));
+            l.setMasterKeyPersisted();
         }
-        return handle;
+        return l;
     }
 
     /**
@@ -991,14 +992,14 @@ public class Bookie extends Thread {
      * never be unfenced. Fencing a fenced ledger has no effect.
      */
     public void fenceLedger(long ledgerId) throws IOException {
-        LedgerDescriptor handle = getHandle(ledgerId, true);
+        LedgerDescriptor handle = getHandle(ledgerId);
         synchronized (handle) {
             handle.setFenced();
         }
     }
 
     public ByteBuffer readEntry(long ledgerId, long entryId) throws IOException {
-        LedgerDescriptor handle = getHandle(ledgerId, true);
+        LedgerDescriptor handle = getHandle(ledgerId);
         try {
             if (LOG.isTraceEnabled()) {
                 LOG.trace("Reading " + entryId + "@" + ledgerId);

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java
Thu Feb  9 17:14:29 2012
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
  * <b>Header</b> is formated as below:
  * <pre>&lt;magic bytes&gt;&lt;len of master key&gt;&lt;master
key&gt;</pre>
  * <ul>
- * <li>magic bytes: 8 bytes, 'BKLE\0\0\0\0'
+ * <li>magic bytes: 4 bytes, 'BKLE', version: 4 bytes
  * <li>len of master key: indicates length of master key. -1 means no master key stored
in header.
  * <li>master key: master key
  * </ul>
@@ -54,78 +54,86 @@ class FileInfo {
 
     private FileChannel fc;
     private final File lf;
+    byte[] masterKey;
+
     /**
      * The fingerprint of a ledger index file
      */
-    private byte header[] = "BKLE\0\0\0\0".getBytes();
+    static final public int signature = ByteBuffer.wrap("BKLE".getBytes()).getInt();
+    static final public int headerVersion = 0;
+
     static final long START_OF_DATA = 1024;
     private long size;
     private int useCount;
     private boolean isClosed;
-    public FileInfo(File lf) throws IOException {
+
+    public FileInfo(File lf, byte[] masterKey) throws IOException {
         this.lf = lf;
-        fc = new RandomAccessFile(lf, "rws").getChannel();
-        size = fc.size();
-        if (size == 0) {
-            fc.write(ByteBuffer.wrap(header));
-            // write NO_MASTER_KEY, which means there is no master key
-            ByteBuffer buf = ByteBuffer.allocate(4);
-            buf.putInt(NO_MASTER_KEY);
-            buf.flip();
-            fc.write(buf);
-        }
+
+        this.masterKey = masterKey;
     }
 
-    /**
-     * Write master key to index file header
-     *
-     * @param masterKey master key to store
-     * @return void
-     * @throws IOException
-     */
-    synchronized public void writeMasterKey(byte[] masterKey) throws IOException {
-        // write master key
-        if (masterKey == null ||
-            masterKey.length + 4 + header.length > START_OF_DATA) {
-            throw new IOException("master key is more than " + (START_OF_DATA - 4 - header.length));
-        }
+    synchronized public void readHeader() throws IOException {
+        if (lf.exists()) {
+            if (fc != null) {
+                return;
+            }
+
+            fc = new RandomAccessFile(lf, "rw").getChannel();
+            size = fc.size();
 
-        int len = masterKey.length;
-        ByteBuffer lenBuf = ByteBuffer.allocate(4);
-        lenBuf.putInt(len);
-        lenBuf.flip();
-        fc.position(header.length);
-        fc.write(lenBuf);
-        fc.write(ByteBuffer.wrap(masterKey));
+            ByteBuffer bb = ByteBuffer.allocate(1024);
+            while(bb.hasRemaining()) {
+                fc.read(bb);
+            }
+            bb.flip();
+            if (bb.getInt() != signature) {
+                throw new IOException("Missing ledger signature");
+            }
+            int version = bb.getInt();
+            if (version != headerVersion) {
+                throw new IOException("Incompatible ledger version " + version);
+            }
+            int length = bb.getInt();
+            if (length < 0 || length > bb.remaining()) {
+                throw new IOException("Length " + length + " is invalid");
+            }
+            masterKey = new byte[length];
+            bb.get(masterKey);
+        } else {
+            throw new IOException("Ledger index file does not exist");
+        }
     }
 
-    /**
-     * Read master key
-     *
-     * @return master key. null means no master key stored in index header
-     * @throws IOException
-     */
-    synchronized public byte[] readMasterKey() throws IOException {
-        ByteBuffer lenBuf = ByteBuffer.allocate(4);
-        int total = readAbsolute(lenBuf, header.length);
-        if (total != 4) {
-            throw new IOException("Short read during reading master key length");
-        }
-        lenBuf.rewind();
-        int len = lenBuf.getInt();
-        if (len == NO_MASTER_KEY) {
-            return null;
-        }
-
-        byte[] masterKey = new byte[len];
-        total = readAbsolute(ByteBuffer.wrap(masterKey), header.length + 4);
-        if (total != len) {
-            throw new IOException("Short read during reading master key");
+    synchronized private void checkOpen(boolean create) throws IOException {
+        if (fc != null) {
+            return;
+        }
+        boolean exists = lf.exists();
+        if (masterKey == null && !exists) {
+            throw new IOException(lf + " not found");
+        }
+        ByteBuffer bb = ByteBuffer.allocate(1024);
+        if (!exists) { 
+            if (create) {
+                fc = new RandomAccessFile(lf, "rw").getChannel();
+                size = fc.size();
+                if (size == 0) {
+                    bb.putInt(signature);
+                    bb.putInt(headerVersion);
+                    bb.putInt(masterKey.length);
+                    bb.put(masterKey);
+                    bb.rewind();
+                    fc.write(bb);
+                }
+            }
+        } else {
+            readHeader();
         }
-        return masterKey;
     }
 
-    synchronized public long size() {
+    synchronized public long size() throws IOException {
+        checkOpen(false);
         long rc = size-START_OF_DATA;
         if (rc < 0) {
             rc = 0;
@@ -138,6 +146,7 @@ class FileInfo {
     }
 
     private int readAbsolute(ByteBuffer bb, long start) throws IOException {
+        checkOpen(false);
         int total = 0;
         while(bb.remaining() > 0) {
             int rc = fc.read(bb, start);
@@ -153,12 +162,13 @@ class FileInfo {
 
     synchronized public void close() throws IOException {
         isClosed = true;
-        if (useCount == 0) {
+        if (useCount == 0 && fc != null) {
             fc.close();
         }
     }
 
     synchronized public long write(ByteBuffer[] buffs, long position) throws IOException
{
+        checkOpen(true);
         long total = 0;
         try {
             fc.position(position+START_OF_DATA);
@@ -170,6 +180,7 @@ class FileInfo {
                 total += rc;
             }
         } finally {
+            fc.force(true);
             long newsize = position+START_OF_DATA+total;
             if (newsize > size) {
                 size = newsize;
@@ -178,13 +189,18 @@ class FileInfo {
         return total;
     }
 
+    synchronized public byte[] getMasterKey() throws IOException {
+        checkOpen(false);
+        return masterKey;
+    }
+
     synchronized public void use() {
         useCount++;
     }
 
     synchronized public void release() {
         useCount--;
-        if (isClosed && useCount == 0) {
+        if (isClosed && useCount == 0 && fc != null) {
             try {
                 fc.close();
             } catch (IOException e) {
@@ -193,12 +209,7 @@ class FileInfo {
         }
     }
 
-    /**
-     * Getter to a handle on the actual ledger index file.
-     * This is used when we are deleting a ledger and want to physically remove the index
file.
-     */
-    File getFile() {
-        return lf;
+    public boolean delete() {
+        return lf.delete();
     }
-
 }

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java
Thu Feb  9 17:14:29 2012
@@ -50,7 +50,7 @@ class JournalChannel {
 
     int HEADER_SIZE = 8; // 4byte magic word, 4 byte version
     int MIN_COMPAT_JOURNAL_FORMAT_VERSION = 1;
-    int CURRENT_JOURNAL_FORMAT_VERSION = 2;
+    int CURRENT_JOURNAL_FORMAT_VERSION = 3;
 
     public final static long preAllocSize = 4*1024*1024;
     public final static ByteBuffer zeros = ByteBuffer.allocate(512);

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java
Thu Feb  9 17:14:29 2012
@@ -226,7 +226,7 @@ public class LedgerCache {
         return dirs[rand.nextInt(dirs.length)];
     }
 
-    FileInfo getFileInfo(Long ledger, boolean create) throws IOException {
+    FileInfo getFileInfo(Long ledger, byte masterKey[]) throws IOException {
         synchronized(fileInfoCache) {
             FileInfo fi = fileInfoCache.get(ledger);
             if (fi == null) {
@@ -240,7 +240,7 @@ public class LedgerCache {
                     lf = null;
                 }
                 if (lf == null) {
-                    if (!create) {
+                    if (masterKey == null) {
                         throw new Bookie.NoLedgerException(ledger);
                     }
                     File dir = pickDirs(ledgerDirectories);
@@ -256,7 +256,7 @@ public class LedgerCache {
                 if (openLedgers.size() > openFileLimit) {
                     fileInfoCache.remove(openLedgers.removeFirst()).close();
                 }
-                fi = new FileInfo(lf);
+                fi = new FileInfo(lf, masterKey);
                 fileInfoCache.put(ledger, fi);
                 openLedgers.add(ledger);
             }
@@ -272,7 +272,7 @@ public class LedgerCache {
         }
         FileInfo fi = null;
         try {
-            fi = getFileInfo(lep.getLedger(), true);
+            fi = getFileInfo(lep.getLedger(), null);
             long pos = lep.getFirstEntry()*8;
             if (pos >= fi.size()) {
                 lep.zeroPage();
@@ -338,7 +338,7 @@ public class LedgerCache {
                         }
                     });
                     ArrayList<Integer> versions = new ArrayList<Integer>(entries.size());
-                    fi = getFileInfo(l, true);
+                    fi = getFileInfo(l, null);
                     int start = 0;
                     long lastOffset = -1;
                     for(int i = 0; i < entries.size(); i++) {
@@ -427,6 +427,7 @@ public class LedgerCache {
                 LedgerEntryPage lep = new LedgerEntryPage(pageSize, entriesPerPage);
                 lep.setLedger(ledger);
                 lep.setFirstEntry(entry);
+
                 // note, this will not block since it is a new page
                 lep.usePage();
                 pageCount++;
@@ -536,8 +537,8 @@ public class LedgerCache {
         if (LOG.isDebugEnabled())
             LOG.debug("Deleting ledgerId: " + ledgerId);
         // Delete the ledger's index file and close the FileInfo
-        FileInfo fi = getFileInfo(ledgerId, false);
-        fi.getFile().delete();
+        FileInfo fi = getFileInfo(ledgerId, null);
+        fi.delete();
         fi.close();
 
         // Remove it from the active ledger manager

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java
Thu Feb  9 17:14:29 2012
@@ -23,6 +23,7 @@ package org.apache.bookkeeper.bookie;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -43,17 +44,46 @@ public class LedgerDescriptor {
         this.ledgerCache = ledgerCache;
     }
 
-    private ByteBuffer masterKey = null;
+    private byte[] masterKey = null;
     volatile private boolean fenced = false;
+    private boolean masterKeyPersisted = false;
 
-    void setMasterKey(ByteBuffer masterKey) {
-        this.masterKey = masterKey;
+    synchronized boolean isMasterKeyPersisted() {
+        if (masterKeyPersisted) {
+            return true;
+        }
+
+        try {
+            FileInfo fi = ledgerCache.getFileInfo(ledgerId, masterKey);
+            fi.readHeader();
+            masterKeyPersisted = true;
+            return true;
+        } catch (IOException ioe) {
+            return false;
+        }
     }
 
-    boolean cmpMasterKey(ByteBuffer masterKey) {
-        return this.masterKey.equals(masterKey);
+    void setMasterKeyPersisted() {
+        masterKeyPersisted = true;
     }
 
+    void checkAccess(byte masterKey[]) throws BookieException, IOException {
+        if (this.masterKey == null) {
+            FileInfo fi = ledgerCache.getFileInfo(ledgerId, masterKey);
+            try {
+                if (fi == null) {
+                    throw new IOException(ledgerId + " does not exist");
+                }
+                this.masterKey = fi.getMasterKey();
+            } finally {
+                fi.release();
+            }
+        }
+        if (!Arrays.equals(this.masterKey, masterKey)) {
+            throw BookieException.create(BookieException.Code.UnauthorizedAccessException);
+        }
+    } 
+
     private long ledgerId;
     public long getLedgerId() {
         return ledgerId;
@@ -109,7 +139,7 @@ public class LedgerDescriptor {
             long lastEntry = ledgerCache.getLastEntry(ledgerId);
             FileInfo fi = null;
             try {
-                fi = ledgerCache.getFileInfo(ledgerId, false);
+                fi = ledgerCache.getFileInfo(ledgerId, null);
                 long size = fi.size();
                 // we may not have the last entry in the cache
                 if (size > lastEntry*8) {

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
Thu Feb  9 17:14:29 2012
@@ -67,7 +67,7 @@ public class BookieServer implements NIO
     protected BookieServerBean jmxBkServerBean;
 
     public BookieServer(ServerConfiguration conf) 
-            throws IOException, KeeperException, InterruptedException {
+            throws IOException, KeeperException, InterruptedException, BookieException {
         this.conf = conf;
         this.bookie = new Bookie(conf);
 
@@ -261,7 +261,7 @@ public class BookieServer implements NIO
      * @throws InterruptedException
      */
     public static void main(String[] args) 
-            throws IOException, KeeperException, InterruptedException {
+            throws IOException, KeeperException, InterruptedException, BookieException {
         ServerConfiguration conf = null;
         try {
             conf = parseArgs(args);

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
Thu Feb  9 17:14:29 2012
@@ -27,6 +27,7 @@ import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 
+import org.apache.bookkeeper.bookie.BookieException;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.proto.BookieServer;
 import org.slf4j.Logger;
@@ -123,7 +124,7 @@ public class LocalBookKeeper {
         }
     }
     private void runBookies(ServerConfiguration baseConf) 
-            throws IOException, KeeperException, InterruptedException {
+            throws IOException, KeeperException, InterruptedException, BookieException {
         LOG.info("Starting Bookie(s)");
         // Create Bookie Servers (B1, B2, B3)
 
@@ -150,7 +151,7 @@ public class LocalBookKeeper {
     }
 
     public static void main(String[] args)
-            throws IOException, KeeperException, InterruptedException {
+            throws IOException, KeeperException, InterruptedException, BookieException {
         if(args.length < 1) {
             usage();
             System.exit(-1);

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java
Thu Feb  9 17:14:29 2012
@@ -59,8 +59,9 @@ public class BookieJournalTest {
             throws Exception {
         File fn = new File(indexDir, LedgerCache.getLedgerName(ledgerId));
         fn.getParentFile().mkdirs();
-        FileInfo fi = new FileInfo(fn);
-        fi.writeMasterKey(masterKey);
+        FileInfo fi = new FileInfo(fn, masterKey);
+        // force creation of index file
+        fi.write(new ByteBuffer[]{ ByteBuffer.allocate(0) }, 0);
         fi.close();
     }
 

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java
Thu Feb  9 17:14:29 2012
@@ -126,7 +126,7 @@ public class BookieLayoutVersionTest ext
         try {
             Bookie b = new Bookie(newServerConfiguration(
                 BOOKIE_PORT, HOSTPORT, newDirectoryWithoutVersion(),
-                new File[] { newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1) }));
+                new File[] { newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1)
}));
             fail("Shouldn't reach here");
         } catch (IOException ioe) {
             assertTrue("Invalid exception", 
@@ -136,7 +136,7 @@ public class BookieLayoutVersionTest ext
         // test with bad data dir
         try {
             Bookie b = new Bookie(newServerConfiguration(
-                BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION
- 1),
+                BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION
- 1),
                 new File[] { newDirectoryWithoutVersion() }));
             fail("Shouldn't reach here");
         } catch (IOException ioe) {
@@ -147,8 +147,8 @@ public class BookieLayoutVersionTest ext
         // test with both bad        
         try {
             Bookie b = new Bookie(newServerConfiguration(
-                BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION
- 1),
-                new File[] { newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1) }));
+                BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION
- 1),
+                new File[] { newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1)
}));
             fail("Shouldn't reach here");
         } catch (IOException ioe) {
             assertTrue("Invalid exception", 
@@ -162,11 +162,11 @@ public class BookieLayoutVersionTest ext
         try {
             Bookie b = new Bookie(newServerConfiguration(
                     BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION),
-                    new File[] { newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1),
+                    new File[] { newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION
- 1),
                                  newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION),
                                  newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION + 1),
                                  newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION),
-                                 newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1),}));
+                                 newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION
- 1),}));
             fail("Shouldn't reach here");
         } catch (IOException ioe) {
             assertTrue("Invalid exception", 

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java
Thu Feb  9 17:14:29 2012
@@ -50,6 +50,7 @@ import org.apache.bookkeeper.client.Asyn
 import org.apache.bookkeeper.client.BookKeeper.DigestType;
 import org.apache.bookkeeper.proto.BookieServer;
 import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.bookie.BookieException;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
(original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
Thu Feb  9 17:14:29 2012
@@ -36,6 +36,7 @@ import org.apache.bookkeeper.client.Book
 import org.apache.bookkeeper.conf.ClientConfiguration;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.proto.BookieServer;
+import org.apache.bookkeeper.bookie.BookieException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.KeeperException;
@@ -199,7 +200,7 @@ public abstract class BaseTestCase exten
      * @throws IOException
      */
     protected void restartBookies() 
-            throws InterruptedException, IOException, KeeperException {
+            throws InterruptedException, IOException, KeeperException, BookieException {
         restartBookies(null);
     }
 
@@ -207,7 +208,7 @@ public abstract class BaseTestCase exten
      * Restart bookie servers add new configuration settings
      */
     protected void restartBookies(ServerConfiguration newConf)
-            throws InterruptedException, IOException, KeeperException {
+            throws InterruptedException, IOException, KeeperException, BookieException {
         // shut down bookie server
         for (BookieServer server : bs) {
             server.shutdown();
@@ -235,7 +236,7 @@ public abstract class BaseTestCase exten
      * @throws IOException
      */
     protected void startNewBookie(int port)
-            throws IOException, InterruptedException, KeeperException {
+            throws IOException, InterruptedException, KeeperException, BookieException {
         File f = File.createTempFile("bookie", "test");
         tmpDirs.add(f);
         f.delete();
@@ -254,7 +255,7 @@ public abstract class BaseTestCase exten
      *
      */
     private BookieServer startBookie(ServerConfiguration conf)
-            throws IOException, InterruptedException, KeeperException {
+            throws IOException, InterruptedException, KeeperException, BookieException {
         BookieServer server = new BookieServer(conf);
         server.start();
 



Mime
View raw message