Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EC1E9DABC for ; Thu, 25 Oct 2012 19:33:15 +0000 (UTC) Received: (qmail 74383 invoked by uid 500); 25 Oct 2012 19:33:15 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 74344 invoked by uid 500); 25 Oct 2012 19:33:15 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 74336 invoked by uid 99); 25 Oct 2012 19:33:15 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 25 Oct 2012 19:33:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 25 Oct 2012 19:33:12 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7419323888E4; Thu, 25 Oct 2012 19:32:27 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1402287 - in /hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/ Date: Thu, 25 Oct 2012 19:32:27 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121025193227.7419323888E4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Thu Oct 25 19:32:26 2012 New Revision: 1402287 URL: http://svn.apache.org/viewvc?rev=1402287&view=rev Log: HDFS-4092. Update file deletion logic for snapshot so that the current inode is removed from the circular linked list; and if some blocks at the end of the block list no longer belong to any other inode, collect them and update the block list. Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileSnapshot.java hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt Thu Oct 25 19:32:26 2012 @@ -28,3 +28,8 @@ Branch-2802 Snapshot (Unreleased) (szetszwo) HDFS-4097. Provide CLI support for createSnapshot. (Brandon Li via suresh) + + HDFS-4092. Update file deletion logic for snapshot so that the current inode + is removed from the circular linked list; and if some blocks at the end of the + block list no longer belong to any other inode, collect them and update the + block list. (szetszwo) Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Oct 25 19:32:26 2012 @@ -307,7 +307,7 @@ public class FSDirectory implements Clos ) throws IOException, QuotaExceededException { waitForReady(); - final INodeFile src = rootDir.getINodeFile(srcPath); + final INodeFile src = INodeFile.valueOf(rootDir.getNode(srcPath, false), srcPath); INodeFileSnapshot snapshot = new INodeFileSnapshot(src, src.computeFileSize(true)); writeLock(); Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Oct 25 19:32:26 2012 @@ -170,6 +170,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState; import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithLink; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; @@ -1398,6 +1399,10 @@ public class FSNamesystem implements Nam throw new HadoopIllegalArgumentException("concat: target file " + target + " is empty"); } + if (trgInode instanceof INodeFileWithLink) { + throw new HadoopIllegalArgumentException("concat: target file " + + target + " is in a snapshot"); + } long blockSize = trgInode.getPreferredBlockSize(); Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Thu Oct 25 19:32:26 2012 @@ -148,21 +148,6 @@ public class INodeDirectory extends INod return getNode(getPathComponents(path), resolveLink); } - /** @return the INodeFile corresponding to the path. */ - INodeFile getINodeFile(String path) throws FileNotFoundException, - UnresolvedLinkException { - final INode inode = getNode(path, false); - if (inode == null) { - throw new FileNotFoundException("File \"" + path - + "\" not found"); - } - if (!(inode instanceof INodeFile)) { - throw new FileNotFoundException("Path \"" + path - + "\" is not a file"); - } - return (INodeFile)inode; - } - /** * Retrieve existing INodes from a path. If existing is big enough to store * all path components (existing and non-existing), then existing INodes Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Thu Oct 25 19:32:26 2012 @@ -55,7 +55,7 @@ public class INodeFile extends INode imp private long header; - BlockInfo blocks[] = null; + protected BlockInfo[] blocks = null; INodeFile(PermissionStatus permissions, BlockInfo[] blklist, short replication, long modificationTime, @@ -162,7 +162,7 @@ public class INodeFile extends INode imp } @Override - int collectSubtreeBlocksAndClear(List v) { + protected int collectSubtreeBlocksAndClear(List v) { parent = null; if(blocks != null && v != null) { for (BlockInfo blk : blocks) { @@ -192,7 +192,7 @@ public class INodeFile extends INode imp /** Compute file size. * May or may not include BlockInfoUnderConstruction. */ - long computeFileSize(boolean includesBlockInfoUnderConstruction) { + protected long computeFileSize(boolean includesBlockInfoUnderConstruction) { if (blocks == null || blocks.length == 0) { return 0; } Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileSnapshot.java?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileSnapshot.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileSnapshot.java Thu Oct 25 19:32:26 2012 @@ -32,4 +32,11 @@ public class INodeFileSnapshot extends I super(f); this.size = size; } + + @Override + protected long computeFileSize(boolean includesBlockInfoUnderConstruction) { + //ignore includesBlockInfoUnderConstruction + //since files in a snapshot are considered as closed. + return size; + } } Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java?rev=1402287&r1=1402286&r2=1402287&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java Thu Oct 25 19:32:26 2012 @@ -17,7 +17,11 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; +import java.util.List; + import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.namenode.INodeFile; /** @@ -65,4 +69,73 @@ public class INodeFileWithLink extends I } return max; } + + /** + * {@inheritDoc} + * + * Remove the current inode from the circular linked list. + * If some blocks at the end of the block list no longer belongs to + * any other inode, collect them and update the block list. + */ + @Override + protected int collectSubtreeBlocksAndClear(List v) { + if (next == this) { + //this is the only remaining inode. + super.collectSubtreeBlocksAndClear(v); + } else { + //There are other inode(s) using the blocks. + //Compute max file size excluding this and find the last inode. + long max = next.computeFileSize(true); + INodeFileWithLink last = next; + for(INodeFileWithLink i = next.getNext(); i != this; i = i.getNext()) { + final long size = i.computeFileSize(true); + if (size > max) { + max = size; + } + last = i; + } + + collectBlocksBeyondMaxAndClear(max, v); + + //remove this from the circular linked list. + last.next = this.next; + this.next = null; + //clear parent + parent = null; + } + return 1; + } + + private void collectBlocksBeyondMaxAndClear(final long max, final List v) { + if (blocks != null) { + //find the minimum n such that the size of the first n blocks > max + int n = 0; + for(long size = 0; n < blocks.length && max > size; n++) { + size += blocks[n].getNumBytes(); + } + + //starting from block[n], the data is beyond max. + if (n < blocks.length) { + //resize the array. + final BlockInfo[] newBlocks; + if (n == 0) { + newBlocks = null; + } else { + newBlocks = new BlockInfo[n]; + System.arraycopy(blocks, 0, newBlocks, 0, n); + } + for(INodeFileWithLink i = next; i != this; i = i.getNext()) { + i.blocks = newBlocks; + } + + //collect the blocks beyond max. + if (v != null) { + for(; n < blocks.length; n++) { + v.add(blocks[n]); + } + } + } + blocks = null; + } + } }