Return-Path: Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: (qmail 14663 invoked from network); 29 Mar 2011 06:02:12 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 29 Mar 2011 06:02:12 -0000 Received: (qmail 64076 invoked by uid 500); 29 Mar 2011 06:02:12 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 64052 invoked by uid 500); 29 Mar 2011 06:02:11 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 64043 invoked by uid 99); 29 Mar 2011 06:02:11 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 29 Mar 2011 06:02:11 +0000 X-ASF-Spam-Status: No, hits=-1998.0 required=5.0 tests=ALL_TRUSTED,FB_GET_MEDS X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 29 Mar 2011 06:02:08 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 7C1D023888CE; Tue, 29 Mar 2011 06:01:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1086487 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/ src/java/org/apache/hadoop/hdfs/tools/off... Date: Tue, 29 Mar 2011 06:01:48 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110329060148.7C1D023888CE@eris.apache.org> Author: todd Date: Tue Mar 29 06:01:47 2011 New Revision: 1086487 URL: http://svn.apache.org/viewvc?rev=1086487&view=rev Log: HDFS-1521. Persist transaction ID on disk between NN restarts. Contributed by Ivan Kelly and Todd Lipcon Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original) +++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Tue Mar 29 06:01:47 2011 @@ -3,3 +3,6 @@ Changes for HDFS-1073 branch This change list will be merged into the trunk CHANGES.txt when the HDFS-1073 branch is merged. ---------------------------- + +HDFS-1521. Persist transaction ID on disk between NN restarts. + (Ivan Kelly and Todd Lipcon via todd) Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java Tue Mar 29 06:01:47 2011 @@ -88,7 +88,7 @@ public interface FSConstants { // Version is reflected in the data storage file. // Versions are negative. // Decrement LAYOUT_VERSION to define a new version. - public static final int LAYOUT_VERSION = -27; + public static final int LAYOUT_VERSION = -28; // Current version: - // -27: remove intentionally corrupt pre-0.13 image directory + // -28: add persistent transaction IDs } Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java Tue Mar 29 06:01:47 2011 @@ -55,6 +55,7 @@ public class BackupImage extends FSImage /** Is journal spooling in progress */ volatile JSpoolState jsState; + private long lastAppliedTxId = 0; static enum JSpoolState { OFF, @@ -153,6 +154,11 @@ public class BackupImage extends FSImage if(!editLog.isOpen()) editLog.open(); + // set storage fields + storage.setStorageInfo(sig); + storage.setImageDigest(sig.getImageDigest()); + storage.setCheckpointTime(sig.checkpointTime); + FSDirectory fsDir = getFSNamesystem().dir; if(fsDir.isEmpty()) { Iterator itImage @@ -163,6 +169,7 @@ public class BackupImage extends FSImage throw new IOException("Could not locate checkpoint directories"); StorageDirectory sdName = itImage.next(); StorageDirectory sdEdits = itEdits.next(); + getFSDirectoryRootLock().writeLock(); try { // load image under rootDir lock loadFSImage(NNStorage.getStorageFile(sdName, NameNodeFile.IMAGE)); @@ -170,12 +177,8 @@ public class BackupImage extends FSImage getFSDirectoryRootLock().writeUnlock(); } loadFSEdits(sdEdits); + lastAppliedTxId = getEditLog().getLastWrittenTxId(); } - - // set storage fields - storage.setStorageInfo(sig); - storage.setImageDigest(sig.imageDigest); - storage.setCheckpointTime(sig.checkpointTime); } /** @@ -227,7 +230,8 @@ public class BackupImage extends FSImage backupInputStream.setBytes(data); FSEditLogLoader logLoader = new FSEditLogLoader(namesystem); logLoader.loadEditRecords(storage.getLayoutVersion(), - backupInputStream.getDataInputStream(), true); + backupInputStream.getDataInputStream(), true, + lastAppliedTxId + 1); getFSNamesystem().dir.updateCountForINodeWithQuota(); // inefficient! break; case INPROGRESS: @@ -348,12 +352,18 @@ public class BackupImage extends FSImage EditLogFileInputStream edits = new EditLogFileInputStream(jSpoolFile); DataInputStream in = edits.getDataInputStream(); FSEditLogLoader logLoader = new FSEditLogLoader(namesystem); - numEdits += logLoader.loadFSEdits(in, false); + int loaded = logLoader.loadFSEdits(in, false, lastAppliedTxId + 1); + + lastAppliedTxId += loaded; + numEdits += loaded; // first time reached the end of spool jsState = JSpoolState.WAIT; - numEdits += logLoader.loadEditRecords(storage.getLayoutVersion(), - in, true); + loaded = logLoader.loadEditRecords(storage.getLayoutVersion(), + in, true, lastAppliedTxId + 1); + numEdits += loaded; + lastAppliedTxId += loaded; + getFSNamesystem().dir.updateCountForINodeWithQuota(); edits.close(); } Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java Tue Mar 29 06:01:47 2011 @@ -51,15 +51,18 @@ class EditLogBackupOutputStream extends static class JournalRecord { byte op; + long txid; Writable[] args; - JournalRecord(byte op, Writable ... writables) { + JournalRecord(byte op, long txid, Writable ... writables) { this.op = op; + this.txid = txid; this.args = writables; } void write(DataOutputStream out) throws IOException { out.write(op); + out.writeLong(txid); if(args == null) return; for(Writable w : args) @@ -105,8 +108,8 @@ class EditLogBackupOutputStream extends } @Override // EditLogOutputStream - void write(byte op, Writable ... writables) throws IOException { - bufCurrent.add(new JournalRecord(op, writables)); + void write(byte op, long txid, Writable ... writables) throws IOException { + bufCurrent.add(new JournalRecord(op, txid, writables)); } /** Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Tue Mar 29 06:01:47 2011 @@ -81,10 +81,18 @@ class EditLogFileOutputStream extends Ed bufCurrent.write(b); } - /** {@inheritDoc} */ + /** + * Write a transaction to the stream. The serialization format is: + *
    + *
  • the opcode (byte)
    • + *
  • the transaction id (long)
    • + *
  • the actual Writables for the transaction
    • + *
+ * */ @Override - void write(byte op, Writable... writables) throws IOException { + void write(byte op, long txid, Writable... writables) throws IOException { write(op); + bufCurrent.writeLong(txid); for (Writable w : writables) { w.write(bufCurrent); } Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java Tue Mar 29 06:01:47 2011 @@ -46,10 +46,12 @@ implements JournalStream { * an array of Writable arguments. * * @param op operation + * @param txid the transaction ID of this operation * @param writables array of Writable arguments * @throws IOException */ - abstract void write(byte op, Writable ... writables) throws IOException; + abstract void write(byte op, long txid, Writable ... writables) + throws IOException; /** * Create and initialize underlying persistent edits log storage. Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Tue Mar 29 06:01:47 2011 @@ -306,12 +306,19 @@ public class FSEditLog implements NNStor if(getNumEditStreams() == 0) throw new java.lang.IllegalStateException(NO_JOURNAL_STREAMS_WARNING); ArrayList errorStreams = null; + + // Only start a new transaction for OPs which will be persisted to disk. + // Obviously this excludes control op codes. long start = now(); + if (opCode.getOpCode() < FSEditLogOpCodes.OP_JSPOOL_START.getOpCode()) { + start = beginTransaction(); + } + for(EditLogOutputStream eStream : editStreams) { if(!eStream.isOperationSupported(opCode.getOpCode())) continue; try { - eStream.write(opCode.getOpCode(), writables); + eStream.write(opCode.getOpCode(), txid, writables); } catch (IOException ie) { LOG.error("logEdit: removing "+ eStream.getName(), ie); if(errorStreams == null) @@ -320,7 +327,7 @@ public class FSEditLog implements NNStor } } disableAndReportErrorOnStreams(errorStreams); - recordTransaction(start); + endTransaction(start); // check if it is time to schedule an automatic sync if (!shouldForceSync()) { @@ -371,7 +378,8 @@ public class FSEditLog implements NNStor return false; } - private void recordTransaction(long start) { + private long beginTransaction() { + assert Thread.holdsLock(this); // get a new transactionId txid++; @@ -380,7 +388,12 @@ public class FSEditLog implements NNStor // TransactionId id = myTransactionId.get(); id.txid = txid; - + return now(); + } + + private void endTransaction(long start) { + assert Thread.holdsLock(this); + // update statistics long end = now(); numTransactions++; @@ -390,6 +403,21 @@ public class FSEditLog implements NNStor } /** + * Return the transaction ID of the last transaction written to the log. + */ + synchronized long getLastWrittenTxId() { + return txid; + } + + /** + * Set the transaction ID to use for the next transaction written. + */ + synchronized void setNextTxId(long nextTxid) { + assert synctxid <= txid; + txid = nextTxid - 1; + } + + /** * Blocks until all ongoing edits have been synced to disk. * This differs from logSync in that it waits for edits that have been * written by other threads, not just edits from the calling thread. @@ -784,6 +812,8 @@ public class FSEditLog implements NNStor /** * Closes the current edit log and opens edits.new. + * @return the transaction id that will be used as the first transaction + * in the new log */ synchronized void rollEditLog() throws IOException { waitForSyncToFinish(); @@ -1014,7 +1044,7 @@ public class FSEditLog implements NNStor if(getNumEditStreams() == 0) throw new java.lang.IllegalStateException(NO_JOURNAL_STREAMS_WARNING); ArrayList errorStreams = null; - long start = now(); + long start = beginTransaction(); for(EditLogOutputStream eStream : editStreams) { try { eStream.write(data, 0, length); @@ -1026,7 +1056,7 @@ public class FSEditLog implements NNStor } } disableAndReportErrorOnStreams(errorStreams); - recordTransaction(start); + endTransaction(start); } /** Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Tue Mar 29 06:01:47 2011 @@ -53,17 +53,20 @@ public class FSEditLogLoader { * This is where we apply edits that we've been writing to disk all * along. */ - int loadFSEdits(EditLogInputStream edits) throws IOException { + int loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) + throws IOException { DataInputStream in = edits.getDataInputStream(); long startTime = now(); - int numEdits = loadFSEdits(in, true); + int numEdits = loadFSEdits(in, true, expectedStartingTxId); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (now()-startTime)/1000 + " seconds."); return numEdits; } - int loadFSEdits(DataInputStream in, boolean closeOnExit) throws IOException { + int loadFSEdits(DataInputStream in, boolean closeOnExit, + long expectedStartingTxId) + throws IOException { int numEdits = 0; int logVersion = 0; @@ -89,19 +92,19 @@ public class FSEditLogLoader { } assert logVersion <= Storage.LAST_UPGRADABLE_LAYOUT_VERSION : "Unsupported version " + logVersion; - numEdits = loadEditRecords(logVersion, in, false); + + numEdits = loadEditRecords(logVersion, in, false, expectedStartingTxId); } finally { if(closeOnExit) in.close(); } - if (logVersion != FSConstants.LAYOUT_VERSION) // other version - numEdits++; // save this image asap + return numEdits; } @SuppressWarnings("deprecation") int loadEditRecords(int logVersion, DataInputStream in, - boolean closeOnExit) throws IOException { + boolean closeOnExit, long expectedStartingTxId) throws IOException { FSDirectory fsDir = fsNamesys.dir; int numEdits = 0; String clientName = null; @@ -116,6 +119,8 @@ public class FSEditLogLoader { numOpUpdateMasterKey = 0, numOpOther = 0; try { + long txId = expectedStartingTxId - 1; + while (true) { long timestamp = 0; long mtime = 0; @@ -133,6 +138,17 @@ public class FSEditLogLoader { } catch (EOFException e) { break; // no more transactions } + + if (logVersion <= -28) { + // Read the txid + long thisTxId = in.readLong(); + if (thisTxId != txId + 1) { + throw new IOException("Expected transaction ID " + + (txId + 1) + " but got " + thisTxId); + } + txId = thisTxId; + } + numEdits++; switch (opCode) { case OP_ADD: Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Tue Mar 29 06:01:47 2011 @@ -162,7 +162,7 @@ public class FSImage implements NNStorag storage.setUpgradeManager(ns.upgradeManager); } } - + void setCheckpointDirectories(Collection dirs, Collection editsDirs) { checkpointDirs = dirs; @@ -636,16 +636,22 @@ public class FSImage implements NNStorag // // Load in bits // + latestEditsSD.read(); + long editsVersion = storage.getLayoutVersion(); latestNameSD.read(); - needToSave |= loadFSImage(NNStorage.getStorageFile(latestNameSD, - NameNodeFile.IMAGE)); + long imageVersion = storage.getLayoutVersion(); + + loadFSImage(NNStorage.getStorageFile(latestNameSD, NameNodeFile.IMAGE)); // Load latest edits - if (latestNameCheckpointTime > latestEditsCheckpointTime) + if (latestNameCheckpointTime > latestEditsCheckpointTime) { // the image is already current, discard edits needToSave |= true; - else // latestNameCheckpointTime == latestEditsCheckpointTime - needToSave |= (loadFSEdits(latestEditsSD) > 0); + } else { // latestNameCheckpointTime == latestEditsCheckpointTime + needToSave |= loadFSEdits(latestEditsSD); + } + needToSave |= (editsVersion != FSConstants.LAYOUT_VERSION + || imageVersion != FSConstants.LAYOUT_VERSION); return needToSave; } @@ -655,12 +661,11 @@ public class FSImage implements NNStorag * filenames and blocks. Return whether we should * "re-save" and consolidate the edit-logs */ - boolean loadFSImage(File curFile) throws IOException { + void loadFSImage(File curFile) throws IOException { FSImageFormat.Loader loader = new FSImageFormat.Loader( conf, getFSNamesystem()); loader.load(curFile); - // Check that the image digest we loaded matches up with what // we expected MD5Hash readImageMd5 = loader.getLoadedImageMd5(); @@ -671,44 +676,42 @@ public class FSImage implements NNStorag " is corrupt with MD5 checksum of " + readImageMd5 + " but expecting " + storage.getImageDigest()); } - - storage.namespaceID = loader.getLoadedNamespaceID(); - storage.layoutVersion = loader.getLoadedImageVersion(); - - boolean needToSave = - loader.getLoadedImageVersion() != FSConstants.LAYOUT_VERSION; - return needToSave; + storage.setCheckpointTxId(loader.getLoadedImageTxId()); } /** * Load and merge edits from two edits files * * @param sd storage directory - * @return number of edits loaded + * @return true if the image should be re-saved * @throws IOException */ - int loadFSEdits(StorageDirectory sd) throws IOException { + boolean loadFSEdits(StorageDirectory sd) throws IOException { FSEditLogLoader loader = new FSEditLogLoader(namesystem); - int numEdits = 0; EditLogFileInputStream edits = new EditLogFileInputStream(NNStorage.getStorageFile(sd, NameNodeFile.EDITS)); - - numEdits = loader.loadFSEdits(edits); + long startingTxId = storage.getCheckpointTxId() + 1; + long numLoaded = loader.loadFSEdits(edits, startingTxId); + startingTxId += numLoaded; + edits.close(); File editsNew = NNStorage.getStorageFile(sd, NameNodeFile.EDITS_NEW); if (editsNew.exists() && editsNew.length() > 0) { edits = new EditLogFileInputStream(editsNew); - numEdits += loader.loadFSEdits(edits); + numLoaded += loader.loadFSEdits(edits, startingTxId); edits.close(); } // update the counts. getFSNamesystem().dir.updateCountForINodeWithQuota(); - return numEdits; + // update the txid for the edit log + editLog.setNextTxId(storage.getCheckpointTxId() + numLoaded + 1); + + return numLoaded > 0; } /** @@ -719,6 +722,7 @@ public class FSImage implements NNStorag FSImageCompression compression = FSImageCompression.createCompression(conf); saver.save(newFile, getFSNamesystem(), compression); storage.setImageDigest(saver.getSavedDigest()); + storage.setCheckpointTxId(editLog.getLastWrittenTxId()); } /** Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Tue Mar 29 06:01:47 2011 @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.permission.P import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.common.GenerationStamp; +import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; @@ -67,10 +68,8 @@ class FSImageFormat { /** Set to true once a file has been loaded using this loader. */ private boolean loaded = false; - /** The image version of the loaded file */ - private int imgVersion; - /** The namespace ID of the loaded file */ - private int imgNamespaceID; + /** The transaction ID of the last edit represented by the loaded file */ + private long imgTxId; /** The MD5 sum of the loaded file */ private MD5Hash imgDigest; @@ -80,15 +79,6 @@ class FSImageFormat { } /** - * Return the version number of the image that has been loaded. - * @throws IllegalStateException if load() has not yet been called. - */ - int getLoadedImageVersion() { - checkLoaded(); - return imgVersion; - } - - /** * Return the MD5 checksum of the image that has been loaded. * @throws IllegalStateException if load() has not yet been called. */ @@ -97,13 +87,9 @@ class FSImageFormat { return imgDigest; } - /** - * Return the namespace ID of the image that has been loaded. - * @throws IllegalStateException if load() has not yet been called. - */ - int getLoadedNamespaceID() { + long getLoadedImageTxId() { checkLoaded(); - return imgNamespaceID; + return imgTxId; } /** @@ -152,10 +138,14 @@ class FSImageFormat { * it should not contain version and namespace fields */ // read image version: first appeared in version -1 - imgVersion = in.readInt(); + long imgVersion = in.readInt(); + if(getLayoutVersion() != imgVersion) + throw new InconsistentFSStateException(curFile, + "imgVersion " + imgVersion + + " expected to be " + getLayoutVersion()); // read namespaceID: first appeared in version -2 - imgNamespaceID = in.readInt(); + in.readInt(); // read number of files long numFiles = readNumFiles(in); @@ -165,6 +155,15 @@ class FSImageFormat { long genstamp = in.readLong(); namesystem.setGenerationStamp(genstamp); } + + // read the transaction ID of the last edit represented by + // this image + if (imgVersion <= -28) { + imgTxId = in.readLong(); + } else { + imgTxId = 0; + } + // read compression related info FSImageCompression compression; @@ -256,11 +255,12 @@ class FSImageFormat { * @return an inode */ private INode loadINode(DataInputStream in) - throws IOException { + throws IOException { long modificationTime = 0; long atime = 0; long blockSize = 0; + long imgVersion = getLayoutVersion(); short replication = in.readShort(); replication = namesystem.adjustReplication(replication); modificationTime = in.readLong(); @@ -326,7 +326,10 @@ class FSImageFormat { modificationTime, atime, nsQuota, dsQuota, blockSize); } - private void loadDatanodes(DataInputStream in) throws IOException { + private void loadDatanodes(DataInputStream in) + throws IOException { + long imgVersion = getLayoutVersion(); + if (imgVersion > -3) // pre datanode image version return; if (imgVersion <= -12) { @@ -342,6 +345,7 @@ class FSImageFormat { private void loadFilesUnderConstruction(DataInputStream in) throws IOException { FSDirectory fsDir = namesystem.dir; + long imgVersion = getLayoutVersion(); if (imgVersion > -13) // pre lease image version return; int size = in.readInt(); @@ -367,7 +371,10 @@ class FSImageFormat { } } - private void loadSecretManagerState(DataInputStream in) throws IOException { + private void loadSecretManagerState(DataInputStream in) + throws IOException { + long imgVersion = getLayoutVersion(); + if (imgVersion > -23) { //SecretManagerState is not available. //This must not happen if security is turned on. @@ -376,8 +383,14 @@ class FSImageFormat { namesystem.loadSecretManagerState(in); } + private long getLayoutVersion() { + return namesystem.getFSImage().getStorage().getLayoutVersion(); + } + + private long readNumFiles(DataInputStream in) + throws IOException { + long imgVersion = getLayoutVersion(); - private long readNumFiles(DataInputStream in) throws IOException { if (imgVersion <= -16) { return in.readLong(); } else { @@ -472,9 +485,12 @@ class FSImageFormat { DataOutputStream out = new DataOutputStream(fos); try { out.writeInt(FSConstants.LAYOUT_VERSION); - out.writeInt(sourceNamesystem.getFSImage().getStorage().getNamespaceID()); // TODO bad dependency + out.writeInt(sourceNamesystem.getFSImage() + .getStorage().getNamespaceID()); // TODO bad dependency out.writeLong(fsDir.rootDir.numItemsInTree()); out.writeLong(sourceNamesystem.getGenerationStamp()); + long txid = sourceNamesystem.getEditLog().getLastWrittenTxId(); + out.writeLong(txid); // write compression info and set up compressed stream out = compression.writeHeaderAndWrapStream(fos); Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Mar 29 06:01:47 2011 @@ -61,6 +61,7 @@ public class NNStorage extends Storage i private static final Log LOG = LogFactory.getLog(NNStorage.class.getName()); static final String MESSAGE_DIGEST_PROPERTY = "imageMD5Digest"; + static final String CHECKPOINT_TXID_PROPERTY = "checkpointTxId"; // // The filenames used for storing the images @@ -145,6 +146,13 @@ public class NNStorage extends Storage i private long checkpointTime = -1L; // The age of the image /** + * TxId of the last transaction that was included in the most + * recent fsimage file. This does not include any transactions + * that have since been written to the edit log. + */ + protected long checkpointTxId; + + /** * list of failed (and thus removed) storages */ final protected List removedStorageDirs @@ -473,6 +481,20 @@ public class NNStorage extends Storage i } /** + * Set the transaction ID of the last checkpoint + */ + void setCheckpointTxId(long checkpointTxId) { + this.checkpointTxId = checkpointTxId; + } + + /** + * Return the transaction ID of the last checkpoint. + */ + long getCheckpointTxId() { + return checkpointTxId; + } + + /** * Set the current checkpoint time. Writes the new checkpoint * time to all available storage directories. * @param newCpT The new checkpoint time. @@ -671,6 +693,21 @@ public class NNStorage extends Storage i " has image MD5 digest when version is " + layoutVersion); } + String sCheckpointId = props.getProperty(CHECKPOINT_TXID_PROPERTY); + if (layoutVersion <= -28) { + if (sCheckpointId == null) { + throw new InconsistentFSStateException(sd.getRoot(), + "file " + STORAGE_FILE_VERSION + + " does not have the checkpoint transaction id set."); + } + this.checkpointTxId = Long.valueOf(sCheckpointId); + } else if (sCheckpointId != null) { + throw new InconsistentFSStateException(sd.getRoot(), + "file " + STORAGE_FILE_VERSION + + " has checkpoint transaction id when version is " + + layoutVersion); + } + this.setCheckpointTime(readCheckpointTime(sd)); } @@ -702,7 +739,7 @@ public class NNStorage extends Storage i } props.setProperty(MESSAGE_DIGEST_PROPERTY, imageDigest.toString()); - + props.setProperty(CHECKPOINT_TXID_PROPERTY, String.valueOf(checkpointTxId)); writeCheckpointTime(sd); } Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Mar 29 06:01:47 2011 @@ -705,12 +705,14 @@ public class SecondaryNameNode implement sdEdits = it.next(); if (sdEdits == null) throw new IOException("Could not locate checkpoint edits"); + + this.getStorage().setStorageInfo(sig); + this.getStorage().setImageDigest(sig.getImageDigest()); if (loadImage) { - // to avoid assert in loadFSImage() - this.getStorage().layoutVersion = -1; loadFSImage(getStorage().getStorageFile(sdName, NameNodeFile.IMAGE)); } loadFSEdits(sdEdits); + sig.validateStorageInfo(this); saveNamespace(false); } Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java Tue Mar 29 06:01:47 2011 @@ -31,6 +31,7 @@ public enum EditsElement { EDITS_VERSION, RECORD, OPCODE, + TRANSACTION_ID, DATA, // elements in the data part of the editLog records LENGTH, Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Tue Mar 29 06:01:47 2011 @@ -49,7 +49,7 @@ import static org.apache.hadoop.hdfs.too class EditsLoaderCurrent implements EditsLoader { private static int [] supportedVersions = { - -18, -19, -20, -21, -22, -23, -24, -25, -26, -27 }; + -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28 }; private EditsVisitor v; private int editsVersion = 0; @@ -77,7 +77,9 @@ class EditsLoaderCurrent implements Edit * Visit OP_INVALID */ private void visit_OP_INVALID() throws IOException { - ; // nothing to do, this op code has no data + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } } /** @@ -101,6 +103,9 @@ class EditsLoaderCurrent implements Edit */ private void visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes editsOpCode) throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } IntToken opAddLength = v.visitInt(EditsElement.LENGTH); // this happens if the edits is not properly ended (-1 op code), @@ -144,6 +149,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_RENAME_OLD */ private void visit_OP_RENAME_OLD() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitInt( EditsElement.LENGTH); v.visitStringUTF8( EditsElement.SOURCE); v.visitStringUTF8( EditsElement.DESTINATION); @@ -154,6 +163,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_DELETE */ private void visit_OP_DELETE() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitInt( EditsElement.LENGTH); v.visitStringUTF8( EditsElement.PATH); v.visitStringUTF8( EditsElement.TIMESTAMP); @@ -163,6 +176,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_MKDIR */ private void visit_OP_MKDIR() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitInt( EditsElement.LENGTH); v.visitStringUTF8( EditsElement.PATH); v.visitStringUTF8( EditsElement.TIMESTAMP); @@ -181,6 +198,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_SET_REPLICATION */ private void visit_OP_SET_REPLICATION() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitStringUTF8(EditsElement.PATH); v.visitStringUTF8(EditsElement.REPLICATION); } @@ -189,6 +210,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_SET_PERMISSIONS */ private void visit_OP_SET_PERMISSIONS() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitStringUTF8( EditsElement.PATH); v.visitShort( EditsElement.FS_PERMISSIONS); } @@ -197,6 +222,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_SET_OWNER */ private void visit_OP_SET_OWNER() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitStringUTF8(EditsElement.PATH); v.visitStringUTF8(EditsElement.USERNAME); v.visitStringUTF8(EditsElement.GROUPNAME); @@ -206,6 +235,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_SET_GENSTAMP */ private void visit_OP_SET_GENSTAMP() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitLong(EditsElement.GENERATION_STAMP); } @@ -213,6 +246,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_TIMES */ private void visit_OP_TIMES() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitInt( EditsElement.LENGTH); v.visitStringUTF8( EditsElement.PATH); v.visitStringUTF8( EditsElement.MTIME); @@ -223,6 +260,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_SET_QUOTA */ private void visit_OP_SET_QUOTA() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitStringUTF8( EditsElement.PATH); v.visitLong( EditsElement.NS_QUOTA); v.visitLong( EditsElement.DS_QUOTA); @@ -232,6 +273,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_RENAME */ private void visit_OP_RENAME() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + if(editsVersion > -21) { throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_RENAME + " for edit log version " + editsVersion @@ -248,6 +293,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_CONCAT_DELETE */ private void visit_OP_CONCAT_DELETE() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + if(editsVersion > -22) { throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_CONCAT_DELETE @@ -268,6 +317,10 @@ class EditsLoaderCurrent implements Edit * Visit OP_SYMLINK */ private void visit_OP_SYMLINK() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + v.visitInt( EditsElement.LENGTH); v.visitStringUTF8( EditsElement.SOURCE); v.visitStringUTF8( EditsElement.DESTINATION); @@ -287,21 +340,25 @@ class EditsLoaderCurrent implements Edit * Visit OP_GET_DELEGATION_TOKEN */ private void visit_OP_GET_DELEGATION_TOKEN() throws IOException { - if(editsVersion > -24) { - throw new IOException("Unexpected op code " + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + + if(editsVersion > -24) { + throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_GET_DELEGATION_TOKEN + " for edit log version " + editsVersion + " (op code 18 only expected for 24 and later)"); - } - v.visitByte( EditsElement.T_VERSION); - v.visitStringText( EditsElement.T_OWNER); - v.visitStringText( EditsElement.T_RENEWER); - v.visitStringText( EditsElement.T_REAL_USER); - v.visitVLong( EditsElement.T_ISSUE_DATE); - v.visitVLong( EditsElement.T_MAX_DATE); - v.visitVInt( EditsElement.T_SEQUENCE_NUMBER); - v.visitVInt( EditsElement.T_MASTER_KEY_ID); - v.visitStringUTF8( EditsElement.T_EXPIRY_TIME); + } + v.visitByte( EditsElement.T_VERSION); + v.visitStringText( EditsElement.T_OWNER); + v.visitStringText( EditsElement.T_RENEWER); + v.visitStringText( EditsElement.T_REAL_USER); + v.visitVLong( EditsElement.T_ISSUE_DATE); + v.visitVLong( EditsElement.T_MAX_DATE); + v.visitVInt( EditsElement.T_SEQUENCE_NUMBER); + v.visitVInt( EditsElement.T_MASTER_KEY_ID); + v.visitStringUTF8( EditsElement.T_EXPIRY_TIME); } /** @@ -309,22 +366,25 @@ class EditsLoaderCurrent implements Edit */ private void visit_OP_RENEW_DELEGATION_TOKEN() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } - if(editsVersion > -24) { - throw new IOException("Unexpected op code " + if(editsVersion > -24) { + throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_RENEW_DELEGATION_TOKEN + " for edit log version " + editsVersion + " (op code 19 only expected for 24 and later)"); - } - v.visitByte( EditsElement.T_VERSION); - v.visitStringText( EditsElement.T_OWNER); - v.visitStringText( EditsElement.T_RENEWER); - v.visitStringText( EditsElement.T_REAL_USER); - v.visitVLong( EditsElement.T_ISSUE_DATE); - v.visitVLong( EditsElement.T_MAX_DATE); - v.visitVInt( EditsElement.T_SEQUENCE_NUMBER); - v.visitVInt( EditsElement.T_MASTER_KEY_ID); - v.visitStringUTF8( EditsElement.T_EXPIRY_TIME); + } + v.visitByte( EditsElement.T_VERSION); + v.visitStringText( EditsElement.T_OWNER); + v.visitStringText( EditsElement.T_RENEWER); + v.visitStringText( EditsElement.T_REAL_USER); + v.visitVLong( EditsElement.T_ISSUE_DATE); + v.visitVLong( EditsElement.T_MAX_DATE); + v.visitVInt( EditsElement.T_SEQUENCE_NUMBER); + v.visitVInt( EditsElement.T_MASTER_KEY_ID); + v.visitStringUTF8( EditsElement.T_EXPIRY_TIME); } /** @@ -332,21 +392,24 @@ class EditsLoaderCurrent implements Edit */ private void visit_OP_CANCEL_DELEGATION_TOKEN() throws IOException { + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } - if(editsVersion > -24) { - throw new IOException("Unexpected op code " + if(editsVersion > -24) { + throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_CANCEL_DELEGATION_TOKEN + " for edit log version " + editsVersion + " (op code 20 only expected for 24 and later)"); - } - v.visitByte( EditsElement.T_VERSION); - v.visitStringText( EditsElement.T_OWNER); - v.visitStringText( EditsElement.T_RENEWER); - v.visitStringText( EditsElement.T_REAL_USER); - v.visitVLong( EditsElement.T_ISSUE_DATE); - v.visitVLong( EditsElement.T_MAX_DATE); - v.visitVInt( EditsElement.T_SEQUENCE_NUMBER); - v.visitVInt( EditsElement.T_MASTER_KEY_ID); + } + v.visitByte( EditsElement.T_VERSION); + v.visitStringText( EditsElement.T_OWNER); + v.visitStringText( EditsElement.T_RENEWER); + v.visitStringText( EditsElement.T_REAL_USER); + v.visitVLong( EditsElement.T_ISSUE_DATE); + v.visitVLong( EditsElement.T_MAX_DATE); + v.visitVInt( EditsElement.T_SEQUENCE_NUMBER); + v.visitVInt( EditsElement.T_MASTER_KEY_ID); } /** @@ -354,17 +417,20 @@ class EditsLoaderCurrent implements Edit */ private void visit_OP_UPDATE_MASTER_KEY() throws IOException { - - if(editsVersion > -24) { - throw new IOException("Unexpected op code " + if(editsVersion <= -28) { + v.visitLong(EditsElement.TRANSACTION_ID); + } + + if(editsVersion > -24) { + throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_UPDATE_MASTER_KEY + " for edit log version " + editsVersion + "(op code 21 only expected for 24 and later)"); - } - v.visitVInt( EditsElement.KEY_ID); - v.visitVLong( EditsElement.KEY_EXPIRY_DATE); - VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH); - v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value); + } + v.visitVInt( EditsElement.KEY_ID); + v.visitVLong( EditsElement.KEY_EXPIRY_DATE); + VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH); + v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value); } private void visitOpCode(FSEditLogOpCodes editsOpCode) Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Tue Mar 29 06:01:47 2011 @@ -121,7 +121,7 @@ class ImageLoaderCurrent implements Imag protected final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int [] versions = - {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27}; + {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28}; private int imageVersion = 0; /* (non-Javadoc) @@ -156,6 +156,10 @@ class ImageLoaderCurrent implements Imag v.visit(ImageElement.GENERATION_STAMP, in.readLong()); + if (imageVersion <= -28) { + v.visit(ImageElement.TRANSACTION_ID, in.readLong()); + } + if (imageVersion <= -25) { boolean isCompressed = in.readBoolean(); v.visit(ImageElement.IS_COMPRESSED, imageVersion); Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java Tue Mar 29 06:01:47 2011 @@ -71,7 +71,8 @@ abstract class ImageVisitor { NUM_DELEGATION_TOKENS, DELEGATION_TOKENS, DELEGATION_TOKEN_IDENTIFIER, - DELEGATION_TOKEN_EXPIRY_TIME + DELEGATION_TOKEN_EXPIRY_TIME, + TRANSACTION_ID } /** Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Tue Mar 29 06:01:47 2011 @@ -178,6 +178,14 @@ public class TestBackupNode extends Test // backup = startBackupNode(conf, op, 1); waitCheckpointDone(backup); + + for (int i = 0; i < 10; i++) { + writeFile(fileSys, new Path("file_" + i), replication); + } + + backup.doCheckpoint(); + waitCheckpointDone(backup); + } catch(IOException e) { LOG.error("Error in TestBackupNode:", e); assertTrue(e.getLocalizedMessage(), false); Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Tue Mar 29 06:01:47 2011 @@ -30,12 +30,15 @@ import org.apache.hadoop.fs.permission.* import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt; + +import org.apache.hadoop.util.StringUtils; import org.mockito.Mockito; @@ -50,6 +53,26 @@ public class TestEditLog extends TestCas static final int NUM_TRANSACTIONS = 100; static final int NUM_THREADS = 100; + /** An edits log with 3 edits from 0.20 - the result of + * a fresh namesystem followed by hadoop fs -touchz /myfile */ + static final byte[] HADOOP20_SOME_EDITS = + StringUtils.hexStringToByte(( + "ffff ffed 0a00 0000 0000 03fa e100 0000" + + "0005 0007 2f6d 7966 696c 6500 0133 000d" + + "3132 3932 3331 3634 3034 3138 3400 0d31" + + "3239 3233 3136 3430 3431 3834 0009 3133" + + "3432 3137 3732 3800 0000 0004 746f 6464" + + "0a73 7570 6572 6772 6f75 7001 a400 1544" + + "4653 436c 6965 6e74 5f2d 3136 3136 3535" + + "3738 3931 000b 3137 322e 3239 2e35 2e33" + + "3209 0000 0005 0007 2f6d 7966 696c 6500" + + "0133 000d 3132 3932 3331 3634 3034 3138" + + "3400 0d31 3239 3233 3136 3430 3431 3834" + + "0009 3133 3432 3137 3732 3800 0000 0004" + + "746f 6464 0a73 7570 6572 6772 6f75 7001" + + "a4ff 0000 0000 0000 0000 0000 0000 0000" + ).replace(" ","")); + // // an object that does a bunch of transactions // @@ -81,6 +104,51 @@ public class TestEditLog extends TestCas } /** + * Test case for an empty edit log from a prior version of Hadoop. + */ + public void testPreTxIdEditLogNoEdits() throws Exception { + FSNamesystem namesys = Mockito.mock(FSNamesystem.class); + int numEdits = testLoad( + StringUtils.hexStringToByte("ffffffed"), // just version number + namesys); + assertEquals(0, numEdits); + } + + /** + * Test case for loading a very simple edit log from a format + * prior to the inclusion of edit transaction IDs in the log. + */ + public void testPreTxidEditLogWithEdits() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster.waitActive(); + final FSNamesystem namesystem = cluster.getNamesystem(); + + int numEdits = testLoad(HADOOP20_SOME_EDITS, namesystem); + assertEquals(3, numEdits); + // Sanity check the edit + HdfsFileStatus fileInfo = namesystem.getFileInfo("/myfile", false); + assertEquals("supergroup", fileInfo.getGroup()); + assertEquals(3, fileInfo.getReplication()); + } finally { + cluster.shutdown(); + } + } + + private int testLoad(byte[] data, FSNamesystem namesys) throws IOException { + FSEditLogLoader loader = new FSEditLogLoader(namesys); + EditLogInputStream mockStream = Mockito.mock(EditLogInputStream.class); + ByteArrayInputStream bais = new ByteArrayInputStream(data); + Mockito.doReturn(new DataInputStream(bais)) + .when(mockStream).getDataInputStream(); + + return loader.loadFSEdits(mockStream, 1); + } + + /** * Tests transaction logging in dfs. */ public void testEditLog() throws IOException { @@ -145,13 +213,13 @@ public class TestEditLog extends TestCas // If there were any corruptions, it is likely that the reading in // of these transactions will throw an exception. // - FSEditLogLoader loader = new FSEditLogLoader(namesystem); for (Iterator it = fsimage.getStorage().dirIterator(NameNodeDirType.EDITS); it.hasNext();) { + FSEditLogLoader loader = new FSEditLogLoader(namesystem); File editFile = NNStorage.getStorageFile(it.next(), NameNodeFile.EDITS); System.out.println("Verifying file: " + editFile); int numEdits = loader.loadFSEdits( - new EditLogFileInputStream(editFile)); + new EditLogFileInputStream(editFile), 1); int numLeases = namesystem.leaseManager.countLease(); System.out.println("Number of outstanding leases " + numLeases); assertEquals(0, numLeases); Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java Tue Mar 29 06:01:47 2011 @@ -186,20 +186,27 @@ public class TestEditLogRace { startTransactionWorkers(namesystem, caughtErr); + long previousLogTxId = 1; + for (int i = 0; i < NUM_ROLLS && caughtErr.get() == null; i++) { try { Thread.sleep(20); } catch (InterruptedException e) {} LOG.info("Starting roll " + i + "."); - editLog.rollEditLog(); - LOG.info("Roll complete " + i + "."); + long nextTxId = 0L; + synchronized (editLog) { + editLog.rollEditLog(); + nextTxId = editLog.getLastWrittenTxId() + 1; + } + LOG.info("Roll " + i + " complete before txid " + nextTxId); - verifyEditLogs(namesystem, fsimage); + verifyEditLogs(namesystem, fsimage, previousLogTxId); LOG.info("Starting purge " + i + "."); editLog.purgeEditLog(); LOG.info("Complete purge " + i + "."); + previousLogTxId = nextTxId; } } finally { stopTransactionWorkers(); @@ -212,7 +219,8 @@ public class TestEditLogRace { } } - private void verifyEditLogs(FSNamesystem namesystem, FSImage fsimage) + private void verifyEditLogs(FSNamesystem namesystem, FSImage fsimage, + long startingTxId) throws IOException { // Verify that we can read in all the transactions that we have written. // If there were any corruptions, it is likely that the reading in @@ -221,8 +229,9 @@ public class TestEditLogRace { fsimage.getStorage().dirIterator(NameNodeDirType.EDITS); it.hasNext();) { File editFile = fsimage.getStorage().getStorageFile(it.next(), NameNodeFile.EDITS); System.out.println("Verifying file: " + editFile); - int numEdits = new FSEditLogLoader(namesystem).loadFSEdits( - new EditLogFileInputStream(editFile)); + FSEditLogLoader loader = new FSEditLogLoader(namesystem); + int numEdits = loader.loadFSEdits(new EditLogFileInputStream(editFile), + startingTxId); System.out.println("Number of edits: " + numEdits); } } @@ -264,14 +273,19 @@ public class TestEditLogRace { namesystem.enterSafeMode(); // Verify edit logs before the save - verifyEditLogs(namesystem, fsimage); + // They should start with the first edit after the checkpoint + verifyEditLogs(namesystem, fsimage, + fsimage.getStorage().getCheckpointTxId() + 1); LOG.info("Save " + i + ": saving namespace"); namesystem.saveNamespace(); LOG.info("Save " + i + ": leaving safemode"); // Verify that edit logs post save are also not corrupt - verifyEditLogs(namesystem, fsimage); + verifyEditLogs(namesystem, fsimage, + fsimage.getStorage().getCheckpointTxId() + 1); + assertEquals(fsimage.getStorage().getCheckpointTxId(), + editLog.getLastWrittenTxId()); namesystem.leaveSafeMode(false); LOG.info("Save " + i + ": complete"); @@ -392,7 +406,9 @@ public class TestEditLogRace { doAnEditThread.join(); assertNull(deferredException.get()); - verifyEditLogs(namesystem, fsimage); + // Since we did one transaction before the saveNamespace, the new + // log starts at txid 2 + verifyEditLogs(namesystem, fsimage, 2); } finally { LOG.info("Closing namesystem"); if(namesystem != null) namesystem.close(); @@ -477,7 +493,9 @@ public class TestEditLogRace { doAnEditThread.join(); assertNull(deferredException.get()); - verifyEditLogs(namesystem, fsimage); + // Since we did one transaction before the saveNamespace, the new + // log starts at txid 2 + verifyEditLogs(namesystem, fsimage, 2); } finally { LOG.info("Closing namesystem"); if(namesystem != null) namesystem.close(); Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Tue Mar 29 06:01:47 2011 @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; +import static org.junit.Assert.*; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.anyObject; @@ -30,6 +31,7 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.permission.FsPermission; @@ -40,6 +42,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.log4j.Level; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -56,6 +59,10 @@ import org.mockito.stubbing.Answer; * */ public class TestSaveNamespace { + static { + ((Log4JLogger)FSImage.LOG).getLogger().setLevel(Level.ALL); + } + private static final Log LOG = LogFactory.getLog(TestSaveNamespace.class); private static class FaultySaveImage implements Answer { @@ -306,7 +313,37 @@ public class TestSaveNamespace { } } } + + @Test + public void testTxIdPersistence() throws Exception { + Configuration conf = getConf(); + NameNode.initMetrics(conf, NamenodeRole.ACTIVE); + NameNode.format(conf); + FSNamesystem fsn = new FSNamesystem(conf); + try { + assertEquals(0, fsn.getEditLog().getLastWrittenTxId()); + doAnEdit(fsn, 1); + assertEquals(1, fsn.getEditLog().getLastWrittenTxId()); + + fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fsn.saveNamespace(); + + // Shut down and restart + fsn.getFSImage().close(); + fsn.close(); + fsn = null; + + fsn = new FSNamesystem(conf); + assertEquals(1, fsn.getEditLog().getLastWrittenTxId()); + + } finally { + if (fsn != null) { + fsn.close(); + } + } + } + private void doAnEdit(FSNamesystem fsn, int id) throws IOException { // Make an edit fsn.mkdirs( Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java?rev=1086487&r1=1086486&r2=1086487&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java Tue Mar 29 06:01:47 2011 @@ -134,15 +134,15 @@ public class TestSecurityTokenEditLog ex // If there were any corruptions, it is likely that the reading in // of these transactions will throw an exception. // - FSEditLogLoader loader = new FSEditLogLoader(namesystem); namesystem.getDelegationTokenSecretManager().stopThreads(); int numKeys = namesystem.getDelegationTokenSecretManager().getNumberOfKeys(); for (Iterator it = fsimage.getStorage().dirIterator(NameNodeDirType.EDITS); it.hasNext();) { + FSEditLogLoader loader = new FSEditLogLoader(namesystem); File editFile = fsimage.getStorage().getStorageFile(it.next(), NameNodeFile.EDITS); System.out.println("Verifying file: " + editFile); int numEdits = loader.loadFSEdits( - new EditLogFileInputStream(editFile)); + new EditLogFileInputStream(editFile), 1); assertTrue("Verification for " + editFile + " failed. " + "Expected " + (NUM_THREADS * opsPerTrans * NUM_TRANSACTIONS + numKeys) + " transactions. "+ "Found " + numEdits + " transactions.",