Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1A94210401 for ; Wed, 8 Jan 2014 14:52:14 +0000 (UTC) Received: (qmail 56558 invoked by uid 500); 8 Jan 2014 14:36:55 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 56488 invoked by uid 500); 8 Jan 2014 14:36:51 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 56284 invoked by uid 99); 8 Jan 2014 14:36:43 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Jan 2014 14:36:43 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Jan 2014 14:36:36 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 75F9B2388B42; Wed, 8 Jan 2014 14:36:16 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1556552 [3/4] - in /hadoop/common/branches/HDFS-5535/hadoop-hdfs-project: hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs/ hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ h... Date: Wed, 08 Jan 2014 14:36:13 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140108143616.75F9B2388B42@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java Wed Jan 8 14:36:09 2014 @@ -53,7 +53,7 @@ abstract class AbstractINodeDiffList removedINodes, boolean countDiffChange) throws QuotaExceededException { - int snapshotIndex = Collections.binarySearch(diffs, snapshot.getId()); + int snapshotIndex = Collections.binarySearch(diffs, snapshot); Quota.Counts counts = Quota.Counts.newInstance(); D removed = null; if (snapshotIndex == 0) { - if (prior != null) { + if (prior != Snapshot.NO_SNAPSHOT_ID) { // there is still snapshot before // set the snapshot to latestBefore - diffs.get(snapshotIndex).setSnapshot(prior); - } else { + diffs.get(snapshotIndex).setSnapshotId(prior); + } else { // there is no snapshot before removed = diffs.remove(0); if (countDiffChange) { counts.add(Quota.NAMESPACE, 1); @@ -96,8 +96,8 @@ abstract class AbstractINodeDiffList 0) { final AbstractINodeDiff previous = diffs.get(snapshotIndex - 1); - if (!previous.getSnapshot().equals(prior)) { - diffs.get(snapshotIndex).setSnapshot(prior); + if (previous.getSnapshotId() != prior) { + diffs.get(snapshotIndex).setSnapshotId(prior); } else { // combine the to-be-removed diff with its previous diff removed = diffs.remove(snapshotIndex); @@ -120,10 +120,10 @@ abstract class AbstractINodeDiffList last = getLast(); - return last == null? null: last.getSnapshot(); + return last == null ? Snapshot.CURRENT_STATE_ID : last.getSnapshotId(); } /** @@ -161,60 +161,49 @@ abstract class AbstractINodeDiffList 0 ? i - 1 : -i - 2; - return diffs.get(priorIndex).getSnapshot(); + return diffs.get(priorIndex).getSnapshotId(); } } else { // the one, or the one before if not existing if (i >= 0) { - return diffs.get(i).getSnapshot(); + return diffs.get(i).getSnapshotId(); } else if (i < -1) { - return diffs.get(-i - 2).getSnapshot(); + return diffs.get(-i - 2).getSnapshotId(); } else { // i == -1 - return null; + return Snapshot.NO_SNAPSHOT_ID; } } } - public final Snapshot getPrior(int snapshotId) { + public final int getPrior(int snapshotId) { return getPrior(snapshotId, false); } /** * Update the prior snapshot. */ - final Snapshot updatePrior(Snapshot snapshot, Snapshot prior) { - int id = snapshot == null ? Snapshot.INVALID_ID : snapshot.getId(); - Snapshot s = getPrior(id, true); - if (s != null && - (prior == null || Snapshot.ID_COMPARATOR.compare(s, prior) > 0)) { - return s; + final int updatePrior(int snapshot, int prior) { + int p = getPrior(snapshot, true); + if (p != Snapshot.CURRENT_STATE_ID + && Snapshot.ID_INTEGER_COMPARATOR.compare(p, prior) > 0) { + return p; } return prior; } - - /** - * @return the diff corresponding to the given snapshot. - * When the diff is null, it means that the current state and - * the corresponding snapshot state are the same. - */ - public final D getDiff(Snapshot snapshot) { - return getDiffById(snapshot == null ? - Snapshot.INVALID_ID : snapshot.getId()); - } - private final D getDiffById(final int snapshotId) { - if (snapshotId == Snapshot.INVALID_ID) { + public final D getDiffById(final int snapshotId) { + if (snapshotId == Snapshot.CURRENT_STATE_ID) { return null; } final int i = Collections.binarySearch(diffs, snapshotId); @@ -234,9 +223,9 @@ abstract class AbstractINodeDiffList= 0) { + && Snapshot.ID_INTEGER_COMPARATOR.compare(last.getSnapshotId(), + latestSnapshotId) >= 0) { return last; } else { try { - return addDiff(latest, currentINode); + return addDiff(latestSnapshotId, currentINode); } catch(NSQuotaExceededException e) { e.setMessagePrefix("Failed to record modification for snapshot"); throw e; @@ -298,10 +288,10 @@ abstract class AbstractINodeDiffList createdList, List deletedList) { - super(snapshot, snapshotINode, posteriorDiff); + DirectoryDiff(int snapshotId, INodeDirectoryAttributes snapshotINode, + DirectoryDiff posteriorDiff, int childrenSize, List createdList, + List deletedList, boolean isSnapshotRoot) { + super(snapshotId, snapshotINode, posteriorDiff); this.childrenSize = childrenSize; this.diff = new ChildrenDiff(createdList, deletedList); + this.isSnapshotRoot = isSnapshotRoot; } ChildrenDiff getChildrenDiff() { return diff; } - - /** Is the inode the root of the snapshot? */ + + void setSnapshotRoot(INodeDirectoryAttributes root) { + this.snapshotINode = root; + this.isSnapshotRoot = true; + } + boolean isSnapshotRoot() { - return snapshotINode == snapshot.getRoot(); + return isSnapshotRoot; } @Override @@ -287,7 +293,7 @@ public class DirectoryWithSnapshotFeatur combined.combinePosterior(d.diff, null); } children = combined.apply2Current(ReadOnlyList.Util.asList( - currentDir.getChildrenList(null))); + currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID))); } return children; } @@ -327,7 +333,7 @@ public class DirectoryWithSnapshotFeatur return null; } else if (d.getPosterior() == null) { // no more posterior diff, get from current inode. - return currentDir.getChild(name, null); + return currentDir.getChild(name, Snapshot.CURRENT_STATE_ID); } } } @@ -342,11 +348,9 @@ public class DirectoryWithSnapshotFeatur writeSnapshot(out); out.writeInt(childrenSize); - // write snapshotINode - if (isSnapshotRoot()) { - out.writeBoolean(true); - } else { - out.writeBoolean(false); + // Write snapshotINode + out.writeBoolean(isSnapshotRoot); + if (!isSnapshotRoot) { if (snapshotINode != null) { out.writeBoolean(true); FSImageSerialization.writeINodeDirectoryAttributes(snapshotINode, out); @@ -373,7 +377,7 @@ public class DirectoryWithSnapshotFeatur extends AbstractINodeDiffList { @Override - DirectoryDiff createDiff(Snapshot snapshot, INodeDirectory currentDir) { + DirectoryDiff createDiff(int snapshot, INodeDirectory currentDir) { return new DirectoryDiff(snapshot, currentDir); } @@ -424,12 +428,13 @@ public class DirectoryWithSnapshotFeatur /** * Destroy a subtree under a DstReference node. */ - public static void destroyDstSubtree(INode inode, final Snapshot snapshot, - final Snapshot prior, final BlocksMapUpdateInfo collectedBlocks, + public static void destroyDstSubtree(INode inode, final int snapshot, + final int prior, final BlocksMapUpdateInfo collectedBlocks, final List removedINodes) throws QuotaExceededException { - Preconditions.checkArgument(prior != null); + Preconditions.checkArgument(prior != Snapshot.NO_SNAPSHOT_ID); if (inode.isReference()) { - if (inode instanceof INodeReference.WithName && snapshot != null) { + if (inode instanceof INodeReference.WithName + && snapshot != Snapshot.CURRENT_STATE_ID) { // this inode has been renamed before the deletion of the DstReference // subtree inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes, @@ -447,18 +452,18 @@ public class DirectoryWithSnapshotFeatur DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature(); if (sf != null) { DirectoryDiffList diffList = sf.getDiffs(); - DirectoryDiff priorDiff = diffList.getDiff(prior); - if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) { + DirectoryDiff priorDiff = diffList.getDiffById(prior); + if (priorDiff != null && priorDiff.getSnapshotId() == prior) { List dList = priorDiff.diff.getList(ListType.DELETED); excludedNodes = cloneDiffList(dList); } - if (snapshot != null) { + if (snapshot != Snapshot.CURRENT_STATE_ID) { diffList.deleteSnapshotDiff(snapshot, prior, dir, collectedBlocks, removedINodes, true); } - priorDiff = diffList.getDiff(prior); - if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) { + priorDiff = diffList.getDiffById(prior); + if (priorDiff != null && priorDiff.getSnapshotId() == prior) { priorDiff.diff.destroyCreatedList(dir, collectedBlocks, removedINodes); } @@ -478,14 +483,14 @@ public class DirectoryWithSnapshotFeatur * deleted list of prior. * @param inode The inode to clean. * @param post The post snapshot. - * @param prior The prior snapshot. + * @param prior The id of the prior snapshot. * @param collectedBlocks Used to collect blocks for later deletion. * @return Quota usage update. */ private static Quota.Counts cleanDeletedINode(INode inode, - final Snapshot post, final Snapshot prior, + final int post, final int prior, final BlocksMapUpdateInfo collectedBlocks, - final List removedINodes, final boolean countDiffChange) + final List removedINodes, final boolean countDiffChange) throws QuotaExceededException { Quota.Counts counts = Quota.Counts.newInstance(); Deque queue = new ArrayDeque(); @@ -494,7 +499,7 @@ public class DirectoryWithSnapshotFeatur INode topNode = queue.pollFirst(); if (topNode instanceof INodeReference.WithName) { INodeReference.WithName wn = (INodeReference.WithName) topNode; - if (wn.getLastSnapshotId() >= post.getId()) { + if (wn.getLastSnapshotId() >= post) { wn.cleanSubtree(post, prior, collectedBlocks, removedINodes, countDiffChange); } @@ -511,8 +516,8 @@ public class DirectoryWithSnapshotFeatur if (sf != null) { // delete files/dirs created after prior. Note that these // files/dirs, along with inode, were deleted right after post. - DirectoryDiff priorDiff = sf.getDiffs().getDiff(prior); - if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) { + DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior); + if (priorDiff != null && priorDiff.getSnapshotId() == prior) { priorChildrenDiff = priorDiff.getChildrenDiff(); counts.add(priorChildrenDiff.destroyCreatedList(dir, collectedBlocks, removedINodes)); @@ -540,8 +545,8 @@ public class DirectoryWithSnapshotFeatur } /** @return the last snapshot. */ - public Snapshot getLastSnapshot() { - return diffs.getLastSnapshot(); + public int getLastSnapshotId() { + return diffs.getLastSnapshotId(); } /** @return the snapshot diff list. */ @@ -565,11 +570,13 @@ public class DirectoryWithSnapshotFeatur * to make sure that parent is in the given snapshot "latest". */ public boolean addChild(INodeDirectory parent, INode inode, - boolean setModTime, Snapshot latest) throws QuotaExceededException { - ChildrenDiff diff = diffs.checkAndAddLatestSnapshotDiff(latest, parent).diff; + boolean setModTime, int latestSnapshotId) throws QuotaExceededException { + ChildrenDiff diff = diffs.checkAndAddLatestSnapshotDiff(latestSnapshotId, + parent).diff; int undoInfo = diff.create(inode); - final boolean added = parent.addChild(inode, setModTime, null); + final boolean added = parent.addChild(inode, setModTime, + Snapshot.CURRENT_STATE_ID); if (!added) { diff.undoCreate(inode, undoInfo); } @@ -581,7 +588,7 @@ public class DirectoryWithSnapshotFeatur * needs to make sure that parent is in the given snapshot "latest". */ public boolean removeChild(INodeDirectory parent, INode child, - Snapshot latest) throws QuotaExceededException { + int latestSnapshotId) throws QuotaExceededException { // For a directory that is not a renamed node, if isInLatestSnapshot returns // false, the directory is not in the latest snapshot, thus we do not need // to record the removed child in any snapshot. @@ -593,7 +600,8 @@ public class DirectoryWithSnapshotFeatur // directory node cannot be in any snapshot (not in current tree, nor in // previous src tree). Thus we do not need to record the removed child in // any snapshot. - ChildrenDiff diff = diffs.checkAndAddLatestSnapshotDiff(latest, parent).diff; + ChildrenDiff diff = diffs.checkAndAddLatestSnapshotDiff(latestSnapshotId, + parent).diff; UndoInfo undoInfo = diff.delete(child); final boolean removed = parent.removeChild(child); @@ -611,29 +619,29 @@ public class DirectoryWithSnapshotFeatur * for the snapshot and return it. */ public ReadOnlyList getChildrenList(INodeDirectory currentINode, - final Snapshot snapshot) { - final DirectoryDiff diff = diffs.getDiff(snapshot); + final int snapshotId) { + final DirectoryDiff diff = diffs.getDiffById(snapshotId); return diff != null ? diff.getChildrenList(currentINode) : currentINode - .getChildrenList(null); + .getChildrenList(Snapshot.CURRENT_STATE_ID); } public INode getChild(INodeDirectory currentINode, byte[] name, - Snapshot snapshot) { - final DirectoryDiff diff = diffs.getDiff(snapshot); + int snapshotId) { + final DirectoryDiff diff = diffs.getDiffById(snapshotId); return diff != null ? diff.getChild(name, true, currentINode) - : currentINode.getChild(name, null); + : currentINode.getChild(name, Snapshot.CURRENT_STATE_ID); } /** Used to record the modification of a symlink node */ public INode saveChild2Snapshot(INodeDirectory currentINode, - final INode child, final Snapshot latest, final INode snapshotCopy) + final INode child, final int latestSnapshotId, final INode snapshotCopy) throws QuotaExceededException { Preconditions.checkArgument(!child.isDirectory(), "child is a directory, child=%s", child); - Preconditions.checkArgument(latest != null); + Preconditions.checkArgument(latestSnapshotId != Snapshot.CURRENT_STATE_ID); - final DirectoryDiff diff = diffs.checkAndAddLatestSnapshotDiff(latest, - currentINode); + final DirectoryDiff diff = diffs.checkAndAddLatestSnapshotDiff( + latestSnapshotId, currentINode); if (diff.getChild(child.getLocalNameBytes(), false, currentINode) != null) { // it was already saved in the latest snapshot earlier. return child; @@ -656,7 +664,7 @@ public class DirectoryWithSnapshotFeatur public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) { for(DirectoryDiff d : diffs) { for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) { - deleted.computeQuotaUsage(counts, false, Snapshot.INVALID_ID); + deleted.computeQuotaUsage(counts, false, Snapshot.CURRENT_STATE_ID); } } counts.add(Quota.NAMESPACE, diffs.asList().size()); @@ -744,14 +752,14 @@ public class DirectoryWithSnapshotFeatur } public Quota.Counts cleanDirectory(final INodeDirectory currentINode, - final Snapshot snapshot, Snapshot prior, + final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks, final List removedINodes, final boolean countDiffChange) throws QuotaExceededException { Quota.Counts counts = Quota.Counts.newInstance(); Map priorCreated = null; Map priorDeleted = null; - if (snapshot == null) { // delete the current directory + if (snapshot == Snapshot.CURRENT_STATE_ID) { // delete the current directory currentINode.recordModification(prior); // delete everything in created list DirectoryDiff lastDiff = diffs.getLast(); @@ -764,9 +772,9 @@ public class DirectoryWithSnapshotFeatur prior = getDiffs().updatePrior(snapshot, prior); // if there is a snapshot diff associated with prior, we need to record // its original created and deleted list before deleting post - if (prior != null) { - DirectoryDiff priorDiff = this.getDiffs().getDiff(prior); - if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) { + if (prior != Snapshot.NO_SNAPSHOT_ID) { + DirectoryDiff priorDiff = this.getDiffs().getDiffById(prior); + if (priorDiff != null && priorDiff.getSnapshotId() == prior) { List cList = priorDiff.diff.getList(ListType.CREATED); List dList = priorDiff.diff.getList(ListType.DELETED); priorCreated = cloneDiffList(cList); @@ -774,13 +782,13 @@ public class DirectoryWithSnapshotFeatur } } - counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior, currentINode, - collectedBlocks, removedINodes, countDiffChange)); + counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior, + currentINode, collectedBlocks, removedINodes, countDiffChange)); // check priorDiff again since it may be created during the diff deletion - if (prior != null) { - DirectoryDiff priorDiff = this.getDiffs().getDiff(prior); - if (priorDiff != null && priorDiff.getSnapshot().equals(prior)) { + if (prior != Snapshot.NO_SNAPSHOT_ID) { + DirectoryDiff priorDiff = this.getDiffs().getDiffById(prior); + if (priorDiff != null && priorDiff.getSnapshotId() == prior) { // For files/directories created between "prior" and "snapshot", // we need to clear snapshot copies for "snapshot". Note that we must // use null as prior in the cleanSubtree call. Files/directories that @@ -791,8 +799,8 @@ public class DirectoryWithSnapshotFeatur for (INode cNode : priorDiff.getChildrenDiff().getList( ListType.CREATED)) { if (priorCreated.containsKey(cNode)) { - counts.add(cNode.cleanSubtree(snapshot, null, collectedBlocks, - removedINodes, countDiffChange)); + counts.add(cNode.cleanSubtree(snapshot, Snapshot.NO_SNAPSHOT_ID, + collectedBlocks, removedINodes, countDiffChange)); } } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java Wed Jan 8 14:36:09 2014 @@ -38,15 +38,15 @@ public class FileDiff extends /** The file size at snapshot creation time. */ private final long fileSize; - FileDiff(Snapshot snapshot, INodeFile file) { - super(snapshot, null, null); + FileDiff(int snapshotId, INodeFile file) { + super(snapshotId, null, null); fileSize = file.computeFileSize(); } /** Constructor used by FSImage loading */ - FileDiff(Snapshot snapshot, INodeFileAttributes snapshotINode, + FileDiff(int snapshotId, INodeFileAttributes snapshotINode, FileDiff posteriorDiff, long fileSize) { - super(snapshot, snapshotINode, posteriorDiff); + super(snapshotId, snapshotINode, posteriorDiff); this.fileSize = fileSize; } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java Wed Jan 8 14:36:09 2014 @@ -25,8 +25,8 @@ public class FileDiffList extends AbstractINodeDiffList { @Override - FileDiff createDiff(Snapshot snapshot, INodeFile file) { - return new FileDiff(snapshot, file); + FileDiff createDiff(int snapshotId, INodeFile file) { + return new FileDiff(snapshotId, file); } @Override Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java Wed Jan 8 14:36:09 2014 @@ -78,22 +78,22 @@ public class FileWithSnapshotFeature imp return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs; } - public Quota.Counts cleanFile(final INodeFile file, final Snapshot snapshot, - Snapshot prior, final BlocksMapUpdateInfo collectedBlocks, + public Quota.Counts cleanFile(final INodeFile file, final int snapshotId, + int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks, final List removedINodes, final boolean countDiffChange) throws QuotaExceededException { - if (snapshot == null) { + if (snapshotId == Snapshot.CURRENT_STATE_ID) { // delete the current file while the file has snapshot feature if (!isCurrentFileDeleted()) { - file.recordModification(prior); + file.recordModification(priorSnapshotId); deleteCurrentFile(); } collectBlocksAndClear(file, collectedBlocks, removedINodes); return Quota.Counts.newInstance(); } else { // delete the snapshot - prior = getDiffs().updatePrior(snapshot, prior); - return diffs.deleteSnapshotDiff(snapshot, prior, file, collectedBlocks, - removedINodes, countDiffChange); + priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId); + return diffs.deleteSnapshotDiff(snapshotId, priorSnapshotId, file, + collectedBlocks, removedINodes, countDiffChange); } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Wed Jan 8 14:36:09 2014 @@ -206,6 +206,15 @@ public class INodeDirectorySnapshottable return i < 0? null: snapshotsByNames.get(i); } + Snapshot getSnapshotById(int sid) { + for (Snapshot s : snapshotsByNames) { + if (s.getId() == sid) { + return s; + } + } + return null; + } + /** @return {@link #snapshotsByNames} as a {@link ReadOnlyList} */ public ReadOnlyList getSnapshotList() { return ReadOnlyList.Util.asReadOnlyList(snapshotsByNames); @@ -297,13 +306,14 @@ public class INodeDirectorySnapshottable + "snapshot with the same name \"" + Snapshot.getSnapshotName(s) + "\"."); } - final DirectoryDiff d = getDiffs().addDiff(s, this); - d.snapshotINode = s.getRoot(); + final DirectoryDiff d = getDiffs().addDiff(id, this); + d.setSnapshotRoot(s.getRoot()); snapshotsByNames.add(-i - 1, s); //set modification time - updateModificationTime(Time.now(), null); - s.getRoot().setModificationTime(getModificationTime(), null); + updateModificationTime(Time.now(), Snapshot.CURRENT_STATE_ID); + s.getRoot().setModificationTime(getModificationTime(), + Snapshot.CURRENT_STATE_ID); return s; } @@ -326,10 +336,10 @@ public class INodeDirectorySnapshottable + ": the snapshot does not exist."); } else { final Snapshot snapshot = snapshotsByNames.get(i); - Snapshot prior = Snapshot.findLatestSnapshot(this, snapshot); + int prior = Snapshot.findLatestSnapshot(this, snapshot.getId()); try { - Quota.Counts counts = cleanSubtree(snapshot, prior, collectedBlocks, - removedINodes, true); + Quota.Counts counts = cleanSubtree(snapshot.getId(), prior, + collectedBlocks, removedINodes, true); INodeDirectory parent = getParent(); if (parent != null) { // there will not be any WithName node corresponding to the deleted @@ -425,8 +435,9 @@ public class INodeDirectorySnapshottable diffReport.addDirDiff(dir, relativePath, diff); } } - ReadOnlyList children = dir.getChildrenList(diffReport - .isFromEarlier() ? diffReport.to : diffReport.from); + ReadOnlyList children = dir.getChildrenList( + diffReport.isFromEarlier() ? Snapshot.getSnapshotId(diffReport.to) : + Snapshot.getSnapshotId(diffReport.from)); for (INode child : children) { final byte[] name = child.getLocalNameBytes(); if (diff.searchIndex(ListType.CREATED, name) < 0 @@ -454,16 +465,15 @@ public class INodeDirectorySnapshottable * Replace itself with {@link INodeDirectoryWithSnapshot} or * {@link INodeDirectory} depending on the latest snapshot. */ - INodeDirectory replaceSelf(final Snapshot latest, final INodeMap inodeMap) + INodeDirectory replaceSelf(final int latestSnapshotId, final INodeMap inodeMap) throws QuotaExceededException { - if (latest == null) { - Preconditions.checkState( - getDirectoryWithSnapshotFeature().getLastSnapshot() == null, - "latest == null but getLastSnapshot() != null, this=%s", this); + if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) { + Preconditions.checkState(getDirectoryWithSnapshotFeature() + .getLastSnapshotId() == Snapshot.CURRENT_STATE_ID, "this=%s", this); } INodeDirectory dir = replaceSelf4INodeDirectory(inodeMap); - if (latest != null) { - dir.recordModification(latest); + if (latestSnapshotId != Snapshot.CURRENT_STATE_ID) { + dir.recordModification(latestSnapshotId); } return dir; } @@ -475,10 +485,10 @@ public class INodeDirectorySnapshottable @Override public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, - Snapshot snapshot) { + int snapshot) { super.dumpTreeRecursively(out, prefix, snapshot); - if (snapshot == null) { + if (snapshot == Snapshot.CURRENT_STATE_ID) { out.println(); out.print(prefix); @@ -494,7 +504,8 @@ public class INodeDirectorySnapshottable n++; } } - Preconditions.checkState(n == snapshotsByNames.size()); + Preconditions.checkState(n == snapshotsByNames.size(), "#n=" + n + + ", snapshotsByNames.size()=" + snapshotsByNames.size()); out.print(", #snapshot="); out.println(n); @@ -522,8 +533,9 @@ public class INodeDirectorySnapshottable @Override public SnapshotAndINode next() { - final Snapshot s = next.snapshot; - final SnapshotAndINode pair = new SnapshotAndINode(s); + final SnapshotAndINode pair = new SnapshotAndINode(next + .getSnapshotId(), getSnapshotById(next.getSnapshotId()) + .getRoot()); next = findNext(); return pair; } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java Wed Jan 8 14:36:09 2014 @@ -37,7 +37,11 @@ import org.apache.hadoop.hdfs.util.ReadO /** Snapshot of a sub-tree in the namesystem. */ @InterfaceAudience.Private public class Snapshot implements Comparable { - public static final int INVALID_ID = -1; + /** + * This id is used to indicate the current state (vs. snapshots) + */ + public static final int CURRENT_STATE_ID = Integer.MAX_VALUE - 1; + public static final int NO_SNAPSHOT_ID = -1; /** * The pattern for generating the default snapshot name. @@ -61,14 +65,18 @@ public class Snapshot implements Compara .toString(); } - /** - * Get the name of the given snapshot. + /** + * Get the name of the given snapshot. * @param s The given snapshot. * @return The name of the snapshot, or an empty string if {@code s} is null */ static String getSnapshotName(Snapshot s) { return s != null ? s.getRoot().getLocalName() : ""; } + + public static int getSnapshotId(Snapshot s) { + return s == null ? CURRENT_STATE_ID : s.getId(); + } /** * Compare snapshot with IDs, where null indicates the current status thus @@ -78,9 +86,8 @@ public class Snapshot implements Compara = new Comparator() { @Override public int compare(Snapshot left, Snapshot right) { - return ID_INTEGER_COMPARATOR.compare( - left == null? null: left.getId(), - right == null? null: right.getId()); + return ID_INTEGER_COMPARATOR.compare(Snapshot.getSnapshotId(left), + Snapshot.getSnapshotId(right)); } }; @@ -92,12 +99,9 @@ public class Snapshot implements Compara = new Comparator() { @Override public int compare(Integer left, Integer right) { - // null means the current state, thus should be the largest - if (left == null) { - return right == null? 0: 1; - } else { - return right == null? -1: left - right; - } + // Snapshot.CURRENT_STATE_ID means the current state, thus should be the + // largest + return left - right; } }; @@ -108,12 +112,12 @@ public class Snapshot implements Compara * is not null). * * @param inode the given inode that the returned snapshot needs to cover - * @param anchor the returned snapshot should be taken before this snapshot. - * @return the latest snapshot covers the given inode and was taken before the - * the given snapshot (if it is not null). + * @param anchor the returned snapshot should be taken before this given id. + * @return id of the latest snapshot that covers the given inode and was taken + * before the the given snapshot (if it is not null). */ - public static Snapshot findLatestSnapshot(INode inode, Snapshot anchor) { - Snapshot latest = null; + public static int findLatestSnapshot(INode inode, final int anchor) { + int latest = NO_SNAPSHOT_ID; for(; inode != null; inode = inode.getParent()) { if (inode.isDirectory()) { final INodeDirectory dir = inode.asDirectory(); @@ -139,13 +143,13 @@ public class Snapshot implements Compara } @Override - public ReadOnlyList getChildrenList(Snapshot snapshot) { - return getParent().getChildrenList(snapshot); + public ReadOnlyList getChildrenList(int snapshotId) { + return getParent().getChildrenList(snapshotId); } @Override - public INode getChild(byte[] name, Snapshot snapshot) { - return getParent().getChild(name, snapshot); + public INode getChild(byte[] name, int snapshotId) { + return getParent().getChild(name, snapshotId); } @Override Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java Wed Jan 8 14:36:09 2014 @@ -118,7 +118,7 @@ public class SnapshotFSImageFormat { private static FileDiff loadFileDiff(FileDiff posterior, DataInput in, FSImageFormat.Loader loader) throws IOException { - // 1. Read the full path of the Snapshot root to identify the Snapshot + // 1. Read the id of the Snapshot root to identify the Snapshot final Snapshot snapshot = loader.getSnapshot(in); // 2. Load file size @@ -128,7 +128,7 @@ public class SnapshotFSImageFormat { final INodeFileAttributes snapshotINode = in.readBoolean()? loader.loadINodeFileAttributes(in): null; - return new FileDiff(snapshot, snapshotINode, posterior, fileSize); + return new FileDiff(snapshot.getId(), snapshotINode, posterior, fileSize); } /** @@ -149,7 +149,8 @@ public class SnapshotFSImageFormat { } // else go to the next SnapshotDiff } // use the current child - INode currentChild = parent.getChild(createdNodeName, null); + INode currentChild = parent.getChild(createdNodeName, + Snapshot.CURRENT_STATE_ID); if (currentChild == null) { throw new IOException("Cannot find an INode associated with the INode " + DFSUtil.bytes2String(createdNodeName) @@ -295,9 +296,9 @@ public class SnapshotFSImageFormat { // 6. Compose the SnapshotDiff List diffs = parent.getDiffs().asList(); - DirectoryDiff sdiff = new DirectoryDiff(snapshot, snapshotINode, - diffs.isEmpty() ? null : diffs.get(0), - childrenSize, createdList, deletedList); + DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode, + diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList, + deletedList, snapshotINode == snapshot.getRoot()); return sdiff; } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Wed Jan 8 14:36:09 2014 @@ -114,7 +114,7 @@ public class SnapshotManager implements s = (INodeDirectorySnapshottable)d; s.setSnapshotQuota(INodeDirectorySnapshottable.SNAPSHOT_LIMIT); } else { - s = d.replaceSelf4INodeDirectorySnapshottable(iip.getLatestSnapshot(), + s = d.replaceSelf4INodeDirectorySnapshottable(iip.getLatestSnapshotId(), fsdir.getINodeMap()); } addSnapshottable(s); @@ -160,7 +160,7 @@ public class SnapshotManager implements if (s == fsdir.getRoot()) { s.setSnapshotQuota(0); } else { - s.replaceSelf(iip.getLatestSnapshot(), fsdir.getINodeMap()); + s.replaceSelf(iip.getLatestSnapshotId(), fsdir.getINodeMap()); } removeSnapshottable(s); } @@ -324,7 +324,8 @@ public class SnapshotManager implements SnapshottableDirectoryStatus status = new SnapshottableDirectoryStatus( dir.getModificationTime(), dir.getAccessTime(), dir.getFsPermission(), dir.getUserName(), dir.getGroupName(), - dir.getLocalNameBytes(), dir.getId(), dir.getChildrenNum(null), + dir.getLocalNameBytes(), dir.getId(), + dir.getChildrenNum(Snapshot.CURRENT_STATE_ID), dir.getNumSnapshots(), dir.getSnapshotQuota(), dir.getParent() == null ? DFSUtil.EMPTY_BYTES : Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java Wed Jan 8 14:36:09 2014 @@ -21,7 +21,7 @@ package org.apache.hadoop.hdfs.server.pr * Utilization report for a Datanode storage */ public class StorageReport { - private final String storageID; + private final DatanodeStorage storage; private final boolean failed; private final long capacity; private final long dfsUsed; @@ -30,9 +30,9 @@ public class StorageReport { public static final StorageReport[] EMPTY_ARRAY = {}; - public StorageReport(String sid, boolean failed, long capacity, long dfsUsed, - long remaining, long bpUsed) { - this.storageID = sid; + public StorageReport(DatanodeStorage storage, boolean failed, + long capacity, long dfsUsed, long remaining, long bpUsed) { + this.storage = storage; this.failed = failed; this.capacity = capacity; this.dfsUsed = dfsUsed; @@ -40,8 +40,8 @@ public class StorageReport { this.blockPoolUsed = bpUsed; } - public String getStorageID() { - return storageID; + public DatanodeStorage getStorage() { + return storage; } public boolean isFailed() { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto Wed Jan 8 14:36:09 2014 @@ -196,12 +196,13 @@ message HeartbeatRequestProto { } message StorageReportProto { - required string storageUuid = 1; + required string storageUuid = 1 [ deprecated = true ]; optional bool failed = 2 [ default = false ]; optional uint64 capacity = 3 [ default = 0 ]; optional uint64 dfsUsed = 4 [ default = 0 ]; optional uint64 remaining = 5 [ default = 0 ]; optional uint64 blockPoolUsed = 6 [ default = 0 ]; + optional DatanodeStorageProto storage = 7; // supersedes StorageUuid } /** Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Wed Jan 8 14:36:09 2014 @@ -140,6 +140,7 @@ public class MiniDFSCluster { private int nameNodeHttpPort = 0; private final Configuration conf; private int numDataNodes = 1; + private StorageType storageType = StorageType.DEFAULT; private boolean format = true; private boolean manageNameDfsDirs = true; private boolean manageNameDfsSharedDirs = true; @@ -186,6 +187,14 @@ public class MiniDFSCluster { } /** + * Default: StorageType.DEFAULT + */ + public Builder storageType(StorageType type) { + this.storageType = type; + return this; + } + + /** * Default: true */ public Builder format(boolean val) { @@ -341,6 +350,7 @@ public class MiniDFSCluster { initMiniDFSCluster(builder.conf, builder.numDataNodes, + builder.storageType, builder.format, builder.manageNameDfsDirs, builder.manageNameDfsSharedDirs, @@ -592,7 +602,7 @@ public class MiniDFSCluster { String[] racks, String hosts[], long[] simulatedCapacities) throws IOException { this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster - initMiniDFSCluster(conf, numDataNodes, format, + initMiniDFSCluster(conf, numDataNodes, StorageType.DEFAULT, format, manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, racks, hosts, simulatedCapacities, null, true, false, @@ -601,7 +611,7 @@ public class MiniDFSCluster { private void initMiniDFSCluster( Configuration conf, - int numDataNodes, boolean format, boolean manageNameDfsDirs, + int numDataNodes, StorageType storageType, boolean format, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean manageDataDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, String clusterId, @@ -670,7 +680,7 @@ public class MiniDFSCluster { } // Start the DataNodes - startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks, + startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig); waitClusterUp(); //make sure ProxyUsers uses the latest conf @@ -990,6 +1000,19 @@ public class MiniDFSCluster { } } + String makeDataNodeDirs(int dnIndex, StorageType storageType) throws IOException { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < DIRS_PER_DATANODE; ++j) { + File dir = getInstanceStorageDir(dnIndex, j); + dir.mkdirs(); + if (!dir.isDirectory()) { + throw new IOException("Mkdirs failed to create directory for DataNode " + dir); + } + sb.append((j > 0 ? "," : "") + "[" + storageType + "]" + fileAsURI(dir)); + } + return sb.toString(); + } + /** * Modify the config and start up additional DataNodes. The info port for * DataNodes is guaranteed to use a free port. @@ -1052,7 +1075,7 @@ public class MiniDFSCluster { String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile) throws IOException { - startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, hosts, + startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, false, false); } @@ -1066,7 +1089,7 @@ public class MiniDFSCluster { long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig) throws IOException { - startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, hosts, + startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false); } @@ -1098,7 +1121,7 @@ public class MiniDFSCluster { * @throws IllegalStateException if NameNode has been shutdown */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, - boolean manageDfsDirs, StartupOption operation, + StorageType storageType, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, @@ -1154,16 +1177,7 @@ public class MiniDFSCluster { // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - StringBuilder sb = new StringBuilder(); - for (int j = 0; j < DIRS_PER_DATANODE; ++j) { - File dir = getInstanceStorageDir(i, j); - dir.mkdirs(); - if (!dir.isDirectory()) { - throw new IOException("Mkdirs failed to create directory for DataNode " + dir); - } - sb.append((j > 0 ? "," : "") + fileAsURI(dir)); - } - String dirs = sb.toString(); + String dirs = makeDataNodeDirs(i, storageType); dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java Wed Jan 8 14:36:09 2014 @@ -50,7 +50,7 @@ public class MiniDFSClusterWithNodeGroup } public synchronized void startDataNodes(Configuration conf, int numDataNodes, - boolean manageDfsDirs, StartupOption operation, + StorageType storageType, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] nodeGroups, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, @@ -112,15 +112,7 @@ public class MiniDFSClusterWithNodeGroup // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - File dir1 = getInstanceStorageDir(i, 0); - File dir2 = getInstanceStorageDir(i, 1); - dir1.mkdirs(); - dir2.mkdirs(); - if (!dir1.isDirectory() || !dir2.isDirectory()) { - throw new IOException("Mkdirs failed to create directory for DataNode " - + i + ": " + dir1 + " or " + dir2); - } - String dirs = fileAsURI(dir1) + "," + fileAsURI(dir2); + String dirs = makeDataNodeDirs(i, storageType); dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); } @@ -198,7 +190,7 @@ public class MiniDFSClusterWithNodeGroup String[] racks, String[] nodeGroups, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile) throws IOException { - startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, nodeGroups, + startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, nodeGroups, hosts, simulatedCapacities, setupHostsFile, false, false); } @@ -213,13 +205,13 @@ public class MiniDFSClusterWithNodeGroup // This is for initialize from parent class. @Override public synchronized void startDataNodes(Configuration conf, int numDataNodes, - boolean manageDfsDirs, StartupOption operation, + StorageType storageType, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig) throws IOException { - startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, + startDataNodes(conf, numDataNodes, storageType, manageDfsDirs, operation, racks, NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Wed Jan 8 14:36:09 2014 @@ -257,8 +257,10 @@ public class BlockManagerTestUtil { DatanodeDescriptor dnd) { ArrayList reports = new ArrayList(); for (DatanodeStorageInfo storage : dnd.getStorageInfos()) { + DatanodeStorage dns = new DatanodeStorage( + storage.getStorageID(), storage.getState(), storage.getStorageType()); StorageReport report = new StorageReport( - storage.getStorageID(), false, storage.getCapacity(), + dns ,false, storage.getCapacity(), storage.getDfsUsed(), storage.getRemaining(), storage.getBlockPoolUsed()); reports.add(report); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Wed Jan 8 14:36:09 2014 @@ -470,11 +470,14 @@ public class TestJspHelper { BlockManagerTestUtil.updateStorage(dnDesc1, new DatanodeStorage("dnStorage1")); BlockManagerTestUtil.updateStorage(dnDesc2, new DatanodeStorage("dnStorage2")); + DatanodeStorage dns1 = new DatanodeStorage("dnStorage1"); + DatanodeStorage dns2 = new DatanodeStorage("dnStorage2"); + StorageReport[] report1 = new StorageReport[] { - new StorageReport("dnStorage1", false, 1024, 100, 924, 100) + new StorageReport(dns1, false, 1024, 100, 924, 100) }; StorageReport[] report2 = new StorageReport[] { - new StorageReport("dnStorage2", false, 2500, 200, 1848, 200) + new StorageReport(dns2, false, 2500, 200, 1848, 200) }; dnDesc1.updateHeartbeat(report1, 5l, 3l, 10, 2); dnDesc2.updateHeartbeat(report2, 10l, 2l, 20, 1); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Wed Jan 8 14:36:09 2014 @@ -394,8 +394,9 @@ public class SimulatedFSDataset implemen } synchronized StorageReport getStorageReport(String bpid) { - return new StorageReport(getStorageUuid(), false, getCapacity(), - getUsed(), getFree(), map.get(bpid).getUsed()); + return new StorageReport(new DatanodeStorage(getStorageUuid()), + false, getCapacity(), getUsed(), getFree(), + map.get(bpid).getUsed()); } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Wed Jan 8 14:36:09 2014 @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.L import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.util.DataChecksum; import org.junit.After; @@ -186,9 +187,8 @@ public class TestDiskError { // Check permissions on directories in 'dfs.datanode.data.dir' FileSystem localFS = FileSystem.getLocal(conf); for (DataNode dn : cluster.getDataNodes()) { - String[] dataDirs = - dn.getConf().getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY); - for (String dir : dataDirs) { + for (FsVolumeSpi v : dn.getFSDataset().getVolumes()) { + String dir = v.getBasePath(); Path dataDir = new Path(dir); FsPermission actual = localFS.getFileStatus(dataDir).getPermission(); assertEquals("Permission for dir: " + dataDir + ", is " + actual + Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java Wed Jan 8 14:36:09 2014 @@ -43,10 +43,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.LogVerificationAppender; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -109,8 +112,9 @@ public class TestFsDatasetCache { public void setUp() throws Exception { assumeTrue(!Path.WINDOWS); conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS, - 500); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 100); + conf.setLong(DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 500); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, CACHE_CAPACITY); @@ -328,7 +332,7 @@ public class TestFsDatasetCache { // Create some test files that will exceed total cache capacity final int numFiles = 5; - final long fileSize = 15000; + final long fileSize = CACHE_CAPACITY / (numFiles-1); final Path[] testFiles = new Path[numFiles]; final HdfsBlockLocation[][] fileLocs = new HdfsBlockLocation[numFiles][]; @@ -477,4 +481,42 @@ public class TestFsDatasetCache { setHeartbeatResponse(uncacheBlocks(locs)); verifyExpectedCacheUsage(0, 0); } + + @Test(timeout=60000) + public void testUncacheQuiesces() throws Exception { + // Create a file + Path fileName = new Path("/testUncacheQuiesces"); + int fileLen = 4096; + DFSTestUtil.createFile(fs, fileName, fileLen, (short)1, 0xFDFD); + // Cache it + DistributedFileSystem dfs = cluster.getFileSystem(); + dfs.addCachePool(new CachePoolInfo("pool")); + dfs.addCacheDirective(new CacheDirectiveInfo.Builder() + .setPool("pool").setPath(fileName).setReplication((short)3).build()); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name()); + long blocksCached = + MetricsAsserts.getLongCounter("BlocksCached", dnMetrics); + return blocksCached > 0; + } + }, 1000, 30000); + // Uncache it + dfs.removeCacheDirective(1); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name()); + long blocksUncached = + MetricsAsserts.getLongCounter("BlocksUncached", dnMetrics); + return blocksUncached > 0; + } + }, 1000, 30000); + // Make sure that no additional messages were sent + Thread.sleep(10000); + MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name()); + MetricsAsserts.assertCounter("BlocksCached", 1l, dnMetrics); + MetricsAsserts.assertCounter("BlocksUncached", 1l, dnMetrics); + } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Wed Jan 8 14:36:09 2014 @@ -938,7 +938,7 @@ public class NNThroughputBenchmark imple // register datanode dnRegistration = nameNodeProto.registerDatanode(dnRegistration); //first block reports - storage = new DatanodeStorage(dnRegistration.getDatanodeUuid()); + storage = new DatanodeStorage(DatanodeStorage.generateUuid()); final StorageBlockReport[] reports = { new StorageBlockReport(storage, new BlockListAsLongs(null, null).getBlockListAsLongs()) @@ -954,8 +954,8 @@ public class NNThroughputBenchmark imple void sendHeartbeat() throws IOException { // register datanode // TODO:FEDERATION currently a single block pool is supported - StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), - false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; + StorageReport[] rep = { new StorageReport(storage, false, + DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0).getCommands(); if(cmds != null) { @@ -1001,7 +1001,7 @@ public class NNThroughputBenchmark imple @SuppressWarnings("unused") // keep it for future blockReceived benchmark int replicateBlocks() throws IOException { // register datanode - StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), + StorageReport[] rep = { new StorageReport(storage, false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0).getCommands(); @@ -1010,7 +1010,8 @@ public class NNThroughputBenchmark imple if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) { // Send a copy of a block to another datanode BlockCommand bcmd = (BlockCommand)cmd; - return transferBlocks(bcmd.getBlocks(), bcmd.getTargets()); + return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(), + bcmd.getTargetStorageIDs()); } } } @@ -1023,12 +1024,14 @@ public class NNThroughputBenchmark imple * that the blocks have been received. */ private int transferBlocks( Block blocks[], - DatanodeInfo xferTargets[][] + DatanodeInfo xferTargets[][], + String targetStorageIDs[][] ) throws IOException { for(int i = 0; i < blocks.length; i++) { DatanodeInfo blockTargets[] = xferTargets[i]; for(int t = 0; t < blockTargets.length; t++) { DatanodeInfo dnInfo = blockTargets[t]; + String targetStorageID = targetStorageIDs[i][t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, new DataStorage(nsInfo), @@ -1038,7 +1041,7 @@ public class NNThroughputBenchmark imple blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - receivedDNReg.getDatanodeUuid(), rdBlocks) }; + targetStorageID, rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode .getNamesystem().getBlockPoolId(), report); } @@ -1127,7 +1130,7 @@ public class NNThroughputBenchmark imple } // create files - LOG.info("Creating " + nrFiles + " with " + blocksPerFile + " blocks each."); + LOG.info("Creating " + nrFiles + " files with " + blocksPerFile + " blocks each."); FileNameGenerator nameGenerator; nameGenerator = new FileNameGenerator(getBaseDir(), 100); String clientName = getClientName(007); @@ -1161,7 +1164,7 @@ public class NNThroughputBenchmark imple loc.getBlock().getLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - datanodes[dnIdx].dnRegistration.getDatanodeUuid(), rdBlocks) }; + datanodes[dnIdx].storage.getStorageID(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc .getBlock().getBlockPoolId(), report); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java Wed Jan 8 14:36:09 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.na import java.io.File; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.Iterator; import org.apache.commons.logging.Log; @@ -29,25 +28,13 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.Options.Rename; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DFSClientAdapter; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.CachePoolInfo; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; /** * OfflineEditsViewerHelper is a helper class for TestOfflineEditsViewer, @@ -135,151 +122,11 @@ public class OfflineEditsViewerHelper { * OP_CLEAR_NS_QUOTA (12) */ private CheckpointSignature runOperations() throws IOException { - LOG.info("Creating edits by performing fs operations"); // no check, if it's not it throws an exception which is what we want - DistributedFileSystem dfs = - (DistributedFileSystem)cluster.getFileSystem(); - FileContext fc = FileContext.getFileContext(cluster.getURI(0), config); - // OP_ADD 0 - Path pathFileCreate = new Path("/file_create_u\1F431"); - FSDataOutputStream s = dfs.create(pathFileCreate); - // OP_CLOSE 9 - s.close(); - // OP_RENAME_OLD 1 - Path pathFileMoved = new Path("/file_moved"); - dfs.rename(pathFileCreate, pathFileMoved); - // OP_DELETE 2 - dfs.delete(pathFileMoved, false); - // OP_MKDIR 3 - Path pathDirectoryMkdir = new Path("/directory_mkdir"); - dfs.mkdirs(pathDirectoryMkdir); - // OP_ALLOW_SNAPSHOT 29 - dfs.allowSnapshot(pathDirectoryMkdir); - // OP_DISALLOW_SNAPSHOT 30 - dfs.disallowSnapshot(pathDirectoryMkdir); - // OP_CREATE_SNAPSHOT 26 - String ssName = "snapshot1"; - dfs.allowSnapshot(pathDirectoryMkdir); - dfs.createSnapshot(pathDirectoryMkdir, ssName); - // OP_RENAME_SNAPSHOT 28 - String ssNewName = "snapshot2"; - dfs.renameSnapshot(pathDirectoryMkdir, ssName, ssNewName); - // OP_DELETE_SNAPSHOT 27 - dfs.deleteSnapshot(pathDirectoryMkdir, ssNewName); - // OP_SET_REPLICATION 4 - s = dfs.create(pathFileCreate); - s.close(); - dfs.setReplication(pathFileCreate, (short)1); - // OP_SET_PERMISSIONS 7 - Short permission = 0777; - dfs.setPermission(pathFileCreate, new FsPermission(permission)); - // OP_SET_OWNER 8 - dfs.setOwner(pathFileCreate, new String("newOwner"), null); - // OP_CLOSE 9 see above - // OP_SET_GENSTAMP 10 see above - // OP_SET_NS_QUOTA 11 obsolete - // OP_CLEAR_NS_QUOTA 12 obsolete - // OP_TIMES 13 - long mtime = 1285195527000L; // Wed, 22 Sep 2010 22:45:27 GMT - long atime = mtime; - dfs.setTimes(pathFileCreate, mtime, atime); - // OP_SET_QUOTA 14 - dfs.setQuota(pathDirectoryMkdir, 1000L, HdfsConstants.QUOTA_DONT_SET); - // OP_RENAME 15 - fc.rename(pathFileCreate, pathFileMoved, Rename.NONE); - // OP_CONCAT_DELETE 16 - Path pathConcatTarget = new Path("/file_concat_target"); - Path[] pathConcatFiles = new Path[2]; - pathConcatFiles[0] = new Path("/file_concat_0"); - pathConcatFiles[1] = new Path("/file_concat_1"); - - long length = blockSize * 3; // multiple of blocksize for concat - short replication = 1; - long seed = 1; - - DFSTestUtil.createFile(dfs, pathConcatTarget, length, replication, seed); - DFSTestUtil.createFile(dfs, pathConcatFiles[0], length, replication, seed); - DFSTestUtil.createFile(dfs, pathConcatFiles[1], length, replication, seed); - dfs.concat(pathConcatTarget, pathConcatFiles); - // OP_SYMLINK 17 - Path pathSymlink = new Path("/file_symlink"); - fc.createSymlink(pathConcatTarget, pathSymlink, false); - // OP_GET_DELEGATION_TOKEN 18 - // OP_RENEW_DELEGATION_TOKEN 19 - // OP_CANCEL_DELEGATION_TOKEN 20 - // see TestDelegationToken.java - // fake the user to renew token for - final Token[] tokens = dfs.addDelegationTokens("JobTracker", null); - UserGroupInformation longUgi = UserGroupInformation.createRemoteUser( - "JobTracker/foo.com@FOO.COM"); - try { - longUgi.doAs(new PrivilegedExceptionAction() { - @Override - public Object run() throws IOException, InterruptedException { - for (Token token : tokens) { - token.renew(config); - token.cancel(config); - } - return null; - } - }); - } catch(InterruptedException e) { - throw new IOException( - "renewDelegationToken threw InterruptedException", e); - } - // OP_UPDATE_MASTER_KEY 21 - // done by getDelegationTokenSecretManager().startThreads(); - - // OP_ADD_CACHE_POOL 35 - final String pool = "poolparty"; - dfs.addCachePool(new CachePoolInfo(pool)); - // OP_MODIFY_CACHE_POOL 36 - dfs.modifyCachePool(new CachePoolInfo(pool) - .setOwnerName("carlton") - .setGroupName("party") - .setMode(new FsPermission((short)0700)) - .setLimit(1989l)); - // OP_ADD_PATH_BASED_CACHE_DIRECTIVE 33 - long id = dfs.addCacheDirective( - new CacheDirectiveInfo.Builder(). - setPath(new Path("/bar")). - setReplication((short)1). - setPool(pool). - build()); - // OP_MODIFY_PATH_BASED_CACHE_DIRECTIVE 38 - dfs.modifyCacheDirective( - new CacheDirectiveInfo.Builder(). - setId(id). - setPath(new Path("/bar2")). - build()); - // OP_REMOVE_PATH_BASED_CACHE_DIRECTIVE 34 - dfs.removeCacheDirective(id); - // OP_REMOVE_CACHE_POOL 37 - dfs.removeCachePool(pool); - // sync to disk, otherwise we parse partial edits - cluster.getNameNode().getFSImage().getEditLog().logSync(); - - // OP_REASSIGN_LEASE 22 - String filePath = "/hard-lease-recovery-test"; - byte[] bytes = "foo-bar-baz".getBytes(); - DFSClientAdapter.stopLeaseRenewer(dfs); - FSDataOutputStream leaseRecoveryPath = dfs.create(new Path(filePath)); - leaseRecoveryPath.write(bytes); - leaseRecoveryPath.hflush(); - // Set the hard lease timeout to 1 second. - cluster.setLeasePeriod(60 * 1000, 1000); - // wait for lease recovery to complete - LocatedBlocks locatedBlocks; - do { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - LOG.info("Innocuous exception", e); - } - locatedBlocks = DFSClientAdapter.callGetBlockLocations( - cluster.getNameNodeRpc(), filePath, 0L, bytes.length); - } while (locatedBlocks.isUnderConstruction()); + DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); + DFSTestUtil.runOperations(cluster, dfs, cluster.getConfiguration(0), + dfs.getDefaultBlockSize(), 0); // Force a roll so we get an OP_END_LOG_SEGMENT txn return cluster.getNameNodeRpc().rollEditLog(); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Wed Jan 8 14:36:09 2014 @@ -57,17 +57,18 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.LogVerificationAppender; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -81,6 +82,7 @@ import org.apache.hadoop.test.GenericTes import org.apache.hadoop.util.GSet; import org.apache.log4j.Level; import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -603,8 +605,8 @@ public class TestCacheDirectives { * Wait for the NameNode to have an expected number of cached blocks * and replicas. * @param nn NameNode - * @param expectedCachedBlocks - * @param expectedCachedReplicas + * @param expectedCachedBlocks if -1, treat as wildcard + * @param expectedCachedReplicas if -1, treat as wildcard * @throws Exception */ private static void waitForCachedBlocks(NameNode nn, @@ -633,16 +635,18 @@ public class TestCacheDirectives { } finally { namesystem.readUnlock(); } - if ((numCachedBlocks == expectedCachedBlocks) && - (numCachedReplicas == expectedCachedReplicas)) { - return true; - } else { - LOG.info(logString + " cached blocks: have " + numCachedBlocks + - " / " + expectedCachedBlocks + ". " + - "cached replicas: have " + numCachedReplicas + - " / " + expectedCachedReplicas); - return false; + if (expectedCachedBlocks == -1 || + numCachedBlocks == expectedCachedBlocks) { + if (expectedCachedReplicas == -1 || + numCachedReplicas == expectedCachedReplicas) { + return true; + } } + LOG.info(logString + " cached blocks: have " + numCachedBlocks + + " / " + expectedCachedBlocks + ". " + + "cached replicas: have " + numCachedReplicas + + " / " + expectedCachedReplicas); + return false; } }, 500, 60000); } @@ -1351,4 +1355,39 @@ public class TestCacheDirectives { .setExpiration(Expiration.newRelative(RELATIVE_EXPIRY_NEVER - 1)) .build()); } + + @Test(timeout=60000) + public void testExceedsCapacity() throws Exception { + // Create a giant file + final Path fileName = new Path("/exceeds"); + final long fileLen = CACHE_CAPACITY * (NUM_DATANODES*2); + int numCachedReplicas = (int) ((CACHE_CAPACITY*NUM_DATANODES)/BLOCK_SIZE); + DFSTestUtil.createFile(dfs, fileName, fileLen, (short) NUM_DATANODES, + 0xFADED); + // Set up a log appender watcher + final LogVerificationAppender appender = new LogVerificationAppender(); + final Logger logger = Logger.getRootLogger(); + logger.addAppender(appender); + dfs.addCachePool(new CachePoolInfo("pool")); + dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool") + .setPath(fileName).setReplication((short) 1).build()); + waitForCachedBlocks(namenode, -1, numCachedReplicas, + "testExceeds:1"); + // Check that no DNs saw an excess CACHE message + int lines = appender.countLinesWithMessage( + "more bytes in the cache: " + + DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY); + assertEquals("Namenode should not send extra CACHE commands", 0, lines); + // Try creating a file with giant-sized blocks that exceed cache capacity + dfs.delete(fileName, false); + DFSTestUtil.createFile(dfs, fileName, 4096, fileLen, CACHE_CAPACITY * 2, + (short) 1, 0xFADED); + // Nothing will get cached, so just force sleep for a bit + Thread.sleep(4000); + // Still should not see any excess commands + lines = appender.countLinesWithMessage( + "more bytes in the cache: " + + DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY); + assertEquals("Namenode should not send extra CACHE commands", 0, lines); + } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java?rev=1556552&r1=1556551&r2=1556552&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java Wed Jan 8 14:36:09 2014 @@ -140,8 +140,9 @@ public class TestDeadDatanode { // Ensure heartbeat from dead datanode is rejected with a command // that asks datanode to register again - StorageReport[] rep = { new StorageReport(reg.getDatanodeUuid(), false, 0, 0, - 0, 0) }; + StorageReport[] rep = { new StorageReport( + new DatanodeStorage(reg.getDatanodeUuid()), + false, 0, 0, 0, 0) }; DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0) .getCommands(); assertEquals(1, cmd.length);