Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A110B18FB6 for ; Fri, 4 Dec 2015 18:26:12 +0000 (UTC) Received: (qmail 23990 invoked by uid 500); 4 Dec 2015 18:26:12 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 23919 invoked by uid 500); 4 Dec 2015 18:26:12 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 23909 invoked by uid 99); 4 Dec 2015 18:26:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Dec 2015 18:26:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2E530E0061; Fri, 4 Dec 2015 18:26:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lei@apache.org To: common-commits@hadoop.apache.org Message-Id: <4775b1a8d28940b389d979b27e905216@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-9490. MiniDFSCluster should change block generation stamp via FsDatasetTestUtils. (Tony Wu via lei) Date: Fri, 4 Dec 2015 18:26:12 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 a68faf1a0 -> 58714f3b7 HDFS-9490. MiniDFSCluster should change block generation stamp via FsDatasetTestUtils. (Tony Wu via lei) (cherry picked from commit 0ac8fb4b336dd03a037092c9712b962e7ed8f852) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58714f3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58714f3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58714f3b Branch: refs/heads/branch-2 Commit: 58714f3b7357d05604987e1db8f6734a7d2827d9 Parents: a68faf1 Author: Lei Xu Authored: Fri Dec 4 10:24:55 2015 -0800 Committer: Lei Xu Committed: Fri Dec 4 10:26:06 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +++----- .../hdfs/server/datanode/FsDatasetTestUtils.java | 9 +++++++++ .../fsdataset/impl/FsDatasetImplTestUtils.java | 14 ++++++++++++++ .../namenode/ha/TestPendingCorruptDnMessages.java | 2 +- 5 files changed, 30 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/58714f3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index aeffe63..f6523ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -841,6 +841,9 @@ Release 2.8.0 - UNRELEASED HDFS-9485. Make BlockManager#removeFromExcessReplicateMap accept BlockInfo instead of Block. (Mingliang Liu via jing9) + HDFS-9490. MiniDFSCluster should change block generation stamp via + FsDatasetTestUtils. (Tony Wu via lei) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/58714f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 0f4816e..092d9f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -2026,12 +2026,10 @@ public class MiniDFSCluster { getMaterializedReplica(i, blk).truncateMeta(newSize); } - public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk, + public void changeGenStampOfBlock(int dnIndex, ExtendedBlock blk, long newGenStamp) throws IOException { - File blockFile = getBlockFile(dnIndex, blk); - File metaFile = FsDatasetUtil.findMetaFile(blockFile); - return metaFile.renameTo(new File(DatanodeUtil.getMetaName( - blockFile.getAbsolutePath(), newGenStamp))); + getFsDatasetTestUtils(dnIndex) + .changeStoredGenerationStamp(blk, newGenStamp); } /* http://git-wip-us.apache.org/repos/asf/hadoop/blob/58714f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java index 07fb7ce..fd47705 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java @@ -242,4 +242,13 @@ public interface FsDatasetTestUtils { * Get the persistently stored generation stamp. */ long getStoredGenerationStamp(ExtendedBlock block) throws IOException; + + /** + * Change the persistently stored generation stamp. + * @param block the block whose generation stamp will be changed + * @param newGenStamp the new generation stamp + * @throws IOException + */ + void changeStoredGenerationStamp(ExtendedBlock block, long newGenStamp) + throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/58714f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java index 8fce163..320ae9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil; import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils; import org.apache.hadoop.hdfs.server.datanode.Replica; @@ -47,6 +48,7 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.Random; /** @@ -363,4 +365,16 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { File[] files = FileUtil.listFiles(dir); return FsDatasetUtil.getGenerationStampFromFile(files, f); } + + @Override + public void changeStoredGenerationStamp( + ExtendedBlock block, long newGenStamp) throws IOException { + File blockFile = + dataset.getBlockFile(block.getBlockPoolId(), block.getBlockId()); + File metaFile = FsDatasetUtil.findMetaFile(blockFile); + File newMetaFile = new File( + DatanodeUtil.getMetaName(blockFile.getAbsolutePath(), newGenStamp)); + Files.move(metaFile.toPath(), newMetaFile.toPath(), + StandardCopyOption.ATOMIC_MOVE); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/58714f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java index 443500c..5f116d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java @@ -68,7 +68,7 @@ public class TestPendingCorruptDnMessages { // Change the gen stamp of the block on datanode to go back in time (gen // stamps start at 1000) ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath); - assertTrue(cluster.changeGenStampOfBlock(0, block, 900)); + cluster.changeGenStampOfBlock(0, block, 900); // Run directory dsscanner to update Datanode's volumeMap DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));