Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 35E469A6D for ; Thu, 1 Mar 2012 00:37:34 +0000 (UTC) Received: (qmail 6448 invoked by uid 500); 1 Mar 2012 00:37:34 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 6417 invoked by uid 500); 1 Mar 2012 00:37:34 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 6407 invoked by uid 99); 1 Mar 2012 00:37:34 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 01 Mar 2012 00:37:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 01 Mar 2012 00:37:30 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 3E2C823888FD; Thu, 1 Mar 2012 00:37:10 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1295356 - in /hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/apache/hadoop/hdfs/tools/offlineEdits... Date: Thu, 01 Mar 2012 00:37:09 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120301003710.3E2C823888FD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: todd Date: Thu Mar 1 00:37:09 2012 New Revision: 1295356 URL: http://svn.apache.org/viewvc?rev=1295356&view=rev Log: HDFS-3023. Optimize entries in edits log for persistBlocks call. Contributed by Todd Lipcon. Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Thu Mar 1 00:37:09 2012 @@ -244,3 +244,5 @@ HDFS-2920. fix remaining TODO items. (at HDFS-3027. Implement a simple NN health check. (atm) HDFS-2979. Balancer should use logical uri for creating failover proxy with HA enabled. (atm) + +HDFS-3023. Optimize entries in edits log for persistBlocks call. (todd) Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Thu Mar 1 00:37:09 2012 @@ -91,7 +91,10 @@ public class LayoutVersion { STORED_TXIDS(-37, "Transaction IDs are stored in edits log and image files"), TXID_BASED_LAYOUT(-38, "File names in NN Storage are based on transaction IDs"), EDITLOG_OP_OPTIMIZATION(-39, - "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"); + "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"), + OPTIMIZE_PERSIST_BLOCKS(-40, + "Serialize block lists with delta-encoded variable length ints, " + + "add OP_UPDATE_BLOCKS"); final int lv; final int ancestorLV; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Mar 1 00:37:09 2012 @@ -369,7 +369,7 @@ public class FSDirectory implements Clos writeLock(); try { - fsImage.getEditLog().logOpenFile(path, file); + fsImage.getEditLog().logUpdateBlocks(path, file); if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* FSDirectory.persistBlocks: " +path+" with "+ file.getBlocks().length Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Thu Mar 1 00:37:09 2012 @@ -626,6 +626,13 @@ public class FSEditLog { logEdit(op); } + public void logUpdateBlocks(String path, INodeFileUnderConstruction file) { + UpdateBlocksOp op = UpdateBlocksOp.getInstance() + .setPath(path) + .setBlocks(file.getBlocks()); + logEdit(op); + } + /** * Add create directory record to edit log */ Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Thu Mar 1 00:37:09 2012 @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.blo import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCloseOp; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.BlockListUpdatingOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.CancelDelegationTokenOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ClearNSQuotaOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ConcatDeleteOp; @@ -55,6 +56,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetReplicationOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SymlinkOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; import org.apache.hadoop.hdfs.util.Holder; @@ -242,6 +244,10 @@ public class FSEditLogLoader { // Fall-through for case 2. // Regardless of whether it's a new file or an updated file, // update the block list. + + // Update the salient file attributes. + newFile.setAccessTime(addCloseOp.atime); + newFile.setModificationTimeForce(addCloseOp.mtime); updateBlocks(fsDir, addCloseOp, newFile); break; } @@ -283,6 +289,24 @@ public class FSEditLogLoader { } break; } + case OP_UPDATE_BLOCKS: { + UpdateBlocksOp updateOp = (UpdateBlocksOp)op; + if (FSNamesystem.LOG.isDebugEnabled()) { + FSNamesystem.LOG.debug(op.opCode + ": " + updateOp.path + + " numblocks : " + updateOp.blocks.length); + } + INodeFile oldFile = getINodeFile(fsDir, updateOp.path); + if (oldFile == null) { + throw new IOException( + "Operation trying to update blocks in non-existent file " + + updateOp.path); + } + + // Update in-memory data structures + updateBlocks(fsDir, updateOp, oldFile); + break; + } + case OP_SET_REPLICATION: { SetReplicationOp setReplicationOp = (SetReplicationOp)op; short replication = fsNamesys.getBlockManager().adjustReplication( @@ -472,32 +496,29 @@ public class FSEditLogLoader { * Update in-memory data structures with new block information. * @throws IOException */ - private void updateBlocks(FSDirectory fsDir, AddCloseOp addCloseOp, + private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op, INodeFile file) throws IOException { - - // Update the salient file attributes. - file.setAccessTime(addCloseOp.atime); - file.setModificationTimeForce(addCloseOp.mtime); - // Update its block list BlockInfo[] oldBlocks = file.getBlocks(); + Block[] newBlocks = op.getBlocks(); + String path = op.getPath(); // Are we only updating the last block's gen stamp. - boolean isGenStampUpdate = oldBlocks.length == addCloseOp.blocks.length; + boolean isGenStampUpdate = oldBlocks.length == newBlocks.length; // First, update blocks in common - for (int i = 0; i < oldBlocks.length && i < addCloseOp.blocks.length; i++) { + for (int i = 0; i < oldBlocks.length && i < newBlocks.length; i++) { BlockInfo oldBlock = oldBlocks[i]; - Block newBlock = addCloseOp.blocks[i]; + Block newBlock = newBlocks[i]; - boolean isLastBlock = i == addCloseOp.blocks.length - 1; + boolean isLastBlock = i == newBlocks.length - 1; if (oldBlock.getBlockId() != newBlock.getBlockId() || (oldBlock.getGenerationStamp() != newBlock.getGenerationStamp() && !(isGenStampUpdate && isLastBlock))) { throw new IOException("Mismatched block IDs or generation stamps, " + "attempting to replace block " + oldBlock + " with " + newBlock + - " as block # " + i + "/" + addCloseOp.blocks.length + " of " + - addCloseOp.path); + " as block # " + i + "/" + newBlocks.length + " of " + + path); } oldBlock.setNumBytes(newBlock.getNumBytes()); @@ -506,7 +527,7 @@ public class FSEditLogLoader { oldBlock.setGenerationStamp(newBlock.getGenerationStamp()); if (oldBlock instanceof BlockInfoUnderConstruction && - (!isLastBlock || addCloseOp.opCode == FSEditLogOpCodes.OP_CLOSE)) { + (!isLastBlock || op.shouldCompleteLastBlock())) { changeMade = true; fsNamesys.getBlockManager().forceCompleteBlock( (INodeFileUnderConstruction)file, @@ -520,24 +541,27 @@ public class FSEditLogLoader { } } - if (addCloseOp.blocks.length < oldBlocks.length) { + if (newBlocks.length < oldBlocks.length) { // We're removing a block from the file, e.g. abandonBlock(...) if (!file.isUnderConstruction()) { throw new IOException("Trying to remove a block from file " + - addCloseOp.path + " which is not under construction."); + path + " which is not under construction."); } - if (addCloseOp.blocks.length != oldBlocks.length - 1) { + if (newBlocks.length != oldBlocks.length - 1) { throw new IOException("Trying to remove more than one block from file " - + addCloseOp.path); + + path); } - fsDir.unprotectedRemoveBlock(addCloseOp.path, + fsDir.unprotectedRemoveBlock(path, (INodeFileUnderConstruction)file, oldBlocks[oldBlocks.length - 1]); - } else if (addCloseOp.blocks.length > oldBlocks.length) { + } else if (newBlocks.length > oldBlocks.length) { // We're adding blocks - for (int i = oldBlocks.length; i < addCloseOp.blocks.length; i++) { - Block newBlock = addCloseOp.blocks[i]; + for (int i = oldBlocks.length; i < newBlocks.length; i++) { + Block newBlock = newBlocks[i]; BlockInfo newBI; - if (addCloseOp.opCode == FSEditLogOpCodes.OP_ADD){ + if (!op.shouldCompleteLastBlock()) { + // TODO: shouldn't this only be true for the last block? + // what about an old-version fsync() where fsync isn't called + // until several blocks in? newBI = new BlockInfoUnderConstruction( newBlock, file.getReplication()); } else { Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java Thu Mar 1 00:37:09 2012 @@ -101,6 +101,7 @@ public abstract class FSEditLogOp { new LogSegmentOp(OP_START_LOG_SEGMENT)); instances.put(OP_END_LOG_SEGMENT, new LogSegmentOp(OP_END_LOG_SEGMENT)); + instances.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp()); return instances; } }; @@ -128,8 +129,14 @@ public abstract class FSEditLogOp { abstract void writeFields(DataOutputStream out) throws IOException; + static interface BlockListUpdatingOp { + Block[] getBlocks(); + String getPath(); + boolean shouldCompleteLastBlock(); + } + @SuppressWarnings("unchecked") - static abstract class AddCloseOp extends FSEditLogOp { + static abstract class AddCloseOp extends FSEditLogOp implements BlockListUpdatingOp { int length; String path; short replication; @@ -151,6 +158,10 @@ public abstract class FSEditLogOp { this.path = path; return (T)this; } + + public String getPath() { + return path; + } T setReplication(short replication) { this.replication = replication; @@ -176,6 +187,10 @@ public abstract class FSEditLogOp { this.blocks = blocks; return (T)this; } + + public Block[] getBlocks() { + return blocks; + } T setPermissionStatus(PermissionStatus permissions) { this.permissions = permissions; @@ -347,6 +362,10 @@ public abstract class FSEditLogOp { return (AddOp)opInstances.get().get(OP_ADD); } + public boolean shouldCompleteLastBlock() { + return false; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); @@ -365,6 +384,10 @@ public abstract class FSEditLogOp { return (CloseOp)opInstances.get().get(OP_CLOSE); } + public boolean shouldCompleteLastBlock() { + return true; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); @@ -373,6 +396,68 @@ public abstract class FSEditLogOp { return builder.toString(); } } + + static class UpdateBlocksOp extends FSEditLogOp implements BlockListUpdatingOp { + String path; + Block[] blocks; + + private UpdateBlocksOp() { + super(OP_UPDATE_BLOCKS); + } + + static UpdateBlocksOp getInstance() { + return (UpdateBlocksOp)opInstances.get() + .get(OP_UPDATE_BLOCKS); + } + + + UpdateBlocksOp setPath(String path) { + this.path = path; + return this; + } + + public String getPath() { + return path; + } + + UpdateBlocksOp setBlocks(Block[] blocks) { + this.blocks = blocks; + return this; + } + + public Block[] getBlocks() { + return blocks; + } + + @Override + void writeFields(DataOutputStream out) throws IOException { + FSImageSerialization.writeString(path, out); + FSImageSerialization.writeCompactBlockArray(blocks, out); + } + + @Override + void readFields(DataInputStream in, int logVersion) throws IOException { + path = FSImageSerialization.readString(in); + this.blocks = FSImageSerialization.readCompactBlockArray( + in, logVersion); + } + + @Override + public boolean shouldCompleteLastBlock() { + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("UpdateBlocksOp [path=") + .append(path) + .append(", blocks=") + .append(Arrays.toString(blocks)) + .append("]"); + return sb.toString(); + } + } static class SetReplicationOp extends FSEditLogOp { String path; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Thu Mar 1 00:37:09 2012 @@ -55,7 +55,8 @@ public enum FSEditLogOpCodes { OP_UPDATE_MASTER_KEY ((byte) 21), OP_REASSIGN_LEASE ((byte) 22), OP_END_LOG_SEGMENT ((byte) 23), - OP_START_LOG_SEGMENT ((byte) 24); + OP_START_LOG_SEGMENT ((byte) 24), + OP_UPDATE_BLOCKS ((byte) 25); private byte opCode; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Thu Mar 1 00:37:09 2012 @@ -40,6 +40,7 @@ import org.apache.hadoop.io.LongWritable import org.apache.hadoop.io.ShortWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; /** * Static utility functions for serializing various pieces of data in the correct @@ -277,6 +278,49 @@ public class FSImageSerialization { ustr.getLength(), (byte) Path.SEPARATOR_CHAR); } + + /** + * Write an array of blocks as compactly as possible. This uses + * delta-encoding for the generation stamp and size, following + * the principle that genstamp increases relatively slowly, + * and size is equal for all but the last block of a file. + */ + public static void writeCompactBlockArray( + Block[] blocks, DataOutputStream out) throws IOException { + WritableUtils.writeVInt(out, blocks.length); + Block prev = null; + for (Block b : blocks) { + long szDelta = b.getNumBytes() - + (prev != null ? prev.getNumBytes() : 0); + long gsDelta = b.getGenerationStamp() - + (prev != null ? prev.getGenerationStamp() : 0); + out.writeLong(b.getBlockId()); // blockid is random + WritableUtils.writeVLong(out, szDelta); + WritableUtils.writeVLong(out, gsDelta); + prev = b; + } + } + + public static Block[] readCompactBlockArray( + DataInputStream in, int logVersion) throws IOException { + int num = WritableUtils.readVInt(in); + if (num < 0) { + throw new IOException("Invalid block array length: " + num); + } + Block prev = null; + Block[] ret = new Block[num]; + for (int i = 0; i < num; i++) { + long id = in.readLong(); + long sz = WritableUtils.readVLong(in) + + ((prev != null) ? prev.getNumBytes() : 0); + long gs = WritableUtils.readVLong(in) + + ((prev != null) ? prev.getGenerationStamp() : 0); + ret[i] = new Block(id, sz, gs); + prev = ret[i]; + } + return ret; + } + /** * DatanodeImage is used to store persistent information * about datanodes into the fsImage. Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java Thu Mar 1 00:37:09 2012 @@ -48,6 +48,8 @@ public enum EditsElement { BLOCK_ID, BLOCK_NUM_BYTES, BLOCK_GENERATION_STAMP, + BLOCK_DELTA_NUM_BYTES, // delta-encoded relative to previous block + BLOCK_DELTA_GEN_STAMP, // delta-encoded relative to previous block PERMISSION_STATUS, FS_PERMISSIONS, CLIENT_NAME, Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Thu Mar 1 00:37:09 2012 @@ -41,7 +41,7 @@ import static org.apache.hadoop.hdfs.too class EditsLoaderCurrent implements EditsLoader { private static int[] supportedVersions = { -18, -19, -20, -21, -22, -23, -24, - -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39}; + -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39, -40}; private EditsVisitor v; private int editsVersion = 0; @@ -150,6 +150,25 @@ class EditsLoaderCurrent implements Edit } } + private void visit_OP_UPDATE_BLOCKS() throws IOException { + visitTxId(); + v.visitStringUTF8(EditsElement.PATH); + VIntToken numBlocksToken = v.visitVInt(EditsElement.NUMBLOCKS); + for (int i = 0; i < numBlocksToken.value; i++) { + v.visitEnclosingElement(EditsElement.BLOCK); + + v.visitLong(EditsElement.BLOCK_ID); + if (i == 0) { + v.visitVLong(EditsElement.BLOCK_NUM_BYTES); + v.visitVLong(EditsElement.BLOCK_GENERATION_STAMP); + } else { + v.visitVLong(EditsElement.BLOCK_DELTA_NUM_BYTES); + v.visitVLong(EditsElement.BLOCK_DELTA_GEN_STAMP); + } + v.leaveEnclosingElement(); + } + } + /** * Visit OP_RENAME_OLD */ @@ -521,6 +540,9 @@ class EditsLoaderCurrent implements Edit case OP_START_LOG_SEGMENT: // 24 visit_OP_BEGIN_LOG_SEGMENT(); break; + case OP_UPDATE_BLOCKS: // 25 + visit_OP_UPDATE_BLOCKS(); + break; default: { throw new IOException("Unknown op code " + editsOpCode); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== Binary files - no diff available. Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml?rev=1295356&r1=1295355&r2=1295356&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml Thu Mar 1 00:37:09 2012 @@ -1,34 +1,34 @@ - -38 + -40 24 1 - 1504643968 + -2045328303 21 2 1 - 1304751257518 + 1331096884634 3 - 2FhO + o0v1 - -174778556 + -1521490291 21 3 2 - 1304751257521 + 1331096884637 3 - 77-r + 3WMF - 1565957291 + 65546244 10 @@ -42,11 +42,10 @@ 0 5 - 5 /file_create 1 - 1304060057562 - 1304060057562 + 1330405685834 + 1330405685834 512 0 @@ -54,20 +53,19 @@ supergroup 420 - DFSClient_NONMAPREDUCE_-66857152_1 + DFSClient_NONMAPREDUCE_-2143415023_1 127.0.0.1 - -1854451489 + 179250704 9 6 - 5 /file_create 1 - 1304060057572 - 1304060057562 + 1330405685848 + 1330405685834 512 0 @@ -76,44 +74,41 @@ 420 - 617592855 + -584136658 1 7 - 3 /file_create /file_moved - 1304060057575 + 1330405685852 - 367100554 + -1983534581 2 8 - 2 /file_moved - 1304060057577 + 1330405685857 - 1048346698 + -97648053 3 9 - 3 /directory_mkdir - 1304060057581 - 0 + 1330405685861 + 1330405685861 todd supergroup 493 - 1207240248 + -146811985 10 @@ -127,11 +122,10 @@ 0 11 - 5 /file_create 1 - 1304060057584 - 1304060057584 + 1330405685866 + 1330405685866 512 0 @@ -139,20 +133,19 @@ supergroup 420 - DFSClient_NONMAPREDUCE_-66857152_1 + DFSClient_NONMAPREDUCE_-2143415023_1 127.0.0.1 - 1796314473 + 806955943 9 12 - 5 /file_create 1 - 1304060057588 - 1304060057584 + 1330405685868 + 1330405685866 512 0 @@ -161,7 +154,7 @@ 420 - 1017626905 + 641893387 4 @@ -170,7 +163,7 @@ /file_create 1 - 1842610087 + 24198146 7 @@ -195,12 +188,11 @@ 13 16 - 3 /file_create 1285195527000 1285195527000 - 1428793678 + 1853168961 14 @@ -216,13 +208,12 @@ 15 18 - 3 /file_create /file_moved - 1304060057605 + 1330405685882 AA - -1155144192 + -1235158297 10 @@ -236,11 +227,10 @@ 0 20 - 5 /file_concat_target 1 - 1304060057613 - 1304060057613 + 1330405685889 + 1330405685889 512 0 @@ -248,36 +238,116 @@ supergroup 420 - DFSClient_NONMAPREDUCE_-66857152_1 + DFSClient_NONMAPREDUCE_-2143415023_1 127.0.0.1 - -428545606 + -981119572 - 9 + 10 21 - 5 + 1004 + + -1627007926 + + + 25 + + 22 + /file_concat_target + 1 + + -7144805496741076283 + 0 + 1004 + + + -1131701615 + + + 10 + + 23 + 1005 + + -957035430 + + + 25 + + 24 + /file_concat_target + 2 + + -7144805496741076283 + 512 + 1004 + + + -4125931756867080767 + -512 + 1 + + + -932985519 + + + 10 + + 25 + 1006 + + -1757460878 + + + 25 + + 26 + /file_concat_target + 3 + + -7144805496741076283 + 512 + 1004 + + + -4125931756867080767 + 0 + 1 + + + 1562413691487277050 + -512 + 1 + + + -154090859 + + + 9 + + 27 /file_concat_target 1 - 1304060057694 - 1304060057613 + 1330405685978 + 1330405685889 512 3 - 3459038074990663911 + -7144805496741076283 512 - 1003 + 1004 - -5555244278278879146 + -4125931756867080767 512 - 1003 + 1005 - -6344128791846831740 + 1562413691487277050 512 - 1003 + 1006 todd @@ -285,25 +355,24 @@ 420 - 707995174 + -292633850 10 - 22 - 1004 + 28 + 1007 - -1500977009 + -1431358549 0 - 23 - 5 + 29 /file_concat_0 1 - 1304060057701 - 1304060057701 + 1330405685983 + 1330405685983 512 0 @@ -311,36 +380,116 @@ supergroup 420 - DFSClient_NONMAPREDUCE_-66857152_1 + DFSClient_NONMAPREDUCE_-2143415023_1 127.0.0.1 - -119850856 + -318194869 + + + 10 + + 30 + 1008 + + 156309208 + + + 25 + + 31 + /file_concat_0 + 1 + + 6084289468290363112 + 0 + 1008 + + + -596016492 + + + 10 + + 32 + 1009 + + -1734001394 + + + 25 + + 33 + /file_concat_0 + 2 + + 6084289468290363112 + 512 + 1008 + + + -4219431127125026105 + -512 + 1 + + + 1352178323 + + + 10 + + 34 + 1010 + + 794444850 + + + 25 + + 35 + /file_concat_0 + 3 + + 6084289468290363112 + 512 + 1008 + + + -4219431127125026105 + 0 + 1 + + + -1765119074945211374 + -512 + 1 + + + -1530696539 9 - 24 - 5 + 36 /file_concat_0 1 - 1304060057737 - 1304060057701 + 1330405686013 + 1330405685983 512 3 - 4671949296381030428 + 6084289468290363112 512 - 1004 + 1008 - -844362243522407159 + -4219431127125026105 512 - 1004 + 1009 - 3476886462779656950 + -1765119074945211374 512 - 1004 + 1010 todd @@ -348,25 +497,24 @@ 420 - -766805874 + -2043978220 10 - 25 - 1005 + 37 + 1011 - 238426056 + 1010571629 0 - 26 - 5 + 38 /file_concat_1 1 - 1304060057742 - 1304060057742 + 1330405686017 + 1330405686017 512 0 @@ -374,36 +522,116 @@ supergroup 420 - DFSClient_NONMAPREDUCE_-66857152_1 + DFSClient_NONMAPREDUCE_-2143415023_1 127.0.0.1 - 1156254705 + -501297097 + + + 10 + + 39 + 1012 + + -1934711736 + + + 25 + + 40 + /file_concat_1 + 1 + + -7448471719302683860 + 0 + 1012 + + + -1853122907 + + + 10 + + 41 + 1013 + + 862670668 + + + 25 + + 42 + /file_concat_1 + 2 + + -7448471719302683860 + 512 + 1012 + + + -8051065559769974521 + -512 + 1 + + + -1169706939 + + + 10 + + 43 + 1014 + + -2070661520 + + + 25 + + 44 + /file_concat_1 + 3 + + -7448471719302683860 + 512 + 1012 + + + -8051065559769974521 + 0 + 1 + + + 3808670437711973616 + -512 + 1 + + + -1568093815 9 - 27 - 5 + 45 /file_concat_1 1 - 1304060057764 - 1304060057742 + 1330405686042 + 1330405686017 512 3 - -754893470864399741 + -7448471719302683860 512 - 1005 + 1012 - 1820875380010181049 + -8051065559769974521 512 - 1005 + 1013 - 8266387560744259971 + 3808670437711973616 512 - 1005 + 1014 todd @@ -411,121 +639,194 @@ 420 - -654780301 + -1640101896 16 - 28 - 4 + 46 /file_concat_target + 2 /file_concat_0 /file_concat_1 - 1304060057767 + 1330405686046 - 1273279541 + 2122891157 17 - 29 - 4 + 47 /file_symlink /file_concat_target - 1304060057770 - 1304060057770 + 1330405686051 + 1330405686051 todd supergroup 511 - 1385678569 + -585385283 18 - 30 + 48 0 todd JobTracker - 1304060057773 - 1304664857773 + 1330405686056 + 1331010486056 1 2 - 1304146457773 + 1330492086056 - 913145699 + 791321007 19 - 31 + 49 0 todd JobTracker - 1304060057773 - 1304664857773 + 1330405686056 + 1331010486056 1 2 - 1304146457785 + 1330492086075 - -1772039941 + 649714969 20 - 32 + 50 0 todd JobTracker - 1304060057773 - 1304664857773 + 1330405686056 + 1331010486056 1 2 - 1382094146 + 1190872628 + + + 10 + + 51 + 1015 + + -460593521 0 - 33 - 5 - /reassign-lease-test + 52 + /hard-lease-recovery-test 1 - 1286491964741 - 1286491964741 + 1330405686084 + 1330405686084 512 0 - atm + todd supergroup 420 - DFSClient_871171074 + DFSClient_NONMAPREDUCE_-2143415023_1 127.0.0.1 - 1975140107 + 2093219037 + + + 10 + + 53 + 1016 + + 120488596 + + + 25 + + 54 + /hard-lease-recovery-test + 1 + + -357061736603024522 + 0 + 1016 + + + 2098840974 + + + 25 + + 55 + /hard-lease-recovery-test + 1 + + -357061736603024522 + 0 + 1016 + + + -1794222801 + + + 10 + + 56 + 1017 + + -2123999915 22 - 34 - DFSClient_871171074 - /reassign-lease-test + 57 + DFSClient_NONMAPREDUCE_-2143415023_1 + /hard-lease-recovery-test HDFS_NameNode - 1975140107 + -1841690515 + + + 9 + + 58 + /hard-lease-recovery-test + 1 + 1330405688726 + 1330405686084 + 512 + 1 + + -357061736603024522 + 11 + 1017 + + + todd + supergroup + 420 + + + -218102037 23 - 35 + 59 - 1975140107 + -1616653774 -1