Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D779218D7F for ; Tue, 1 Sep 2015 22:20:51 +0000 (UTC) Received: (qmail 26083 invoked by uid 500); 1 Sep 2015 22:20:51 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 25931 invoked by uid 500); 1 Sep 2015 22:20:51 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 25573 invoked by uid 99); 1 Sep 2015 22:20:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Sep 2015 22:20:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CAC79E0941; Tue, 1 Sep 2015 22:20:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinodkv@apache.org To: common-commits@hadoop.apache.org Date: Tue, 01 Sep 2015 22:20:53 -0000 Message-Id: <796c78e21999470599038c89873d4db1@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [4/4] hadoop git commit: HDFS-7885. Datanode should not trust the generation stamp provided by client. Contributed by Tsz Wo Nicholas Sze. HDFS-7885. Datanode should not trust the generation stamp provided by client. Contributed by Tsz Wo Nicholas Sze. (cherry picked from commit 24db0812be64e83a48ade01fc1eaaeaedad4dec0) (cherry picked from commit 994dadb9ba0a3b87b6548e6e0801eadd26554d55) (cherry picked from commit 0bc5c6495a7feb4365af0ce5fe48fc87b7e1749f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b25491dc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b25491dc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b25491dc Branch: refs/heads/branch-2.6.1 Commit: b25491dc45be92cc3ec157d998a2632387c3952f Parents: 7ffdf7d Author: Jing Zhao Authored: Fri Mar 6 10:55:56 2015 -0800 Committer: Vinod Kumar Vavilapalli Committed: Tue Sep 1 15:17:57 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +++++ .../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 63 ++++++++++++++++++++ 3 files changed, 81 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b25491dc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1811337..d7ff237 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -85,6 +85,9 @@ Release 2.6.1 - UNRELEASED HDFS-7871. NameNodeEditLogRoller can keep printing "Swallowing exception" message. (jing9) + HDFS-7885. Datanode should not trust the generation stamp provided by + client. (Tsz Wo Nicholas Sze via jing9) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b25491dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 0d9f096..0c2337e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2276,6 +2276,21 @@ class FsDatasetImpl implements FsDatasetSpi { @Override // FsDatasetSpi public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block) throws IOException { + synchronized(this) { + final Replica replica = volumeMap.get(block.getBlockPoolId(), + block.getBlockId()); + if (replica == null) { + throw new ReplicaNotFoundException(block); + } + if (replica.getGenerationStamp() < block.getGenerationStamp()) { + throw new IOException( + "Replica generation stamp < block generation stamp, block=" + + block + ", replica=" + replica); + } else if (replica.getGenerationStamp() > block.getGenerationStamp()) { + block.setGenerationStamp(replica.getGenerationStamp()); + } + } + File datafile = getBlockFile(block); File metafile = FsDatasetUtil.getMetaFile(datafile, block.getGenerationStamp()); BlockLocalPathInfo info = new BlockLocalPathInfo(block, http://git-wip-us.apache.org/repos/asf/hadoop/blob/b25491dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java index cb50539..1c4134f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java @@ -30,11 +30,16 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; +import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.junit.Assert; import org.junit.Assume; import org.junit.BeforeClass; @@ -153,4 +158,62 @@ public class TestBlockReaderLocalLegacy { Arrays.equals(orig, buf); cluster.shutdown(); } + + @Test(timeout=20000) + public void testBlockReaderLocalLegacyWithAppend() throws Exception { + final short REPL_FACTOR = 1; + final HdfsConfiguration conf = getConfiguration(null); + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true); + + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + + final DistributedFileSystem dfs = cluster.getFileSystem(); + final Path path = new Path("/testBlockReaderLocalLegacy"); + DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0); + DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR); + + final ClientDatanodeProtocol proxy; + final Token token; + final ExtendedBlock originalBlock; + final long originalGS; + { + final LocatedBlock lb = cluster.getNameNode().getRpcServer() + .getBlockLocations(path.toString(), 0, 1).get(0); + proxy = DFSUtil.createClientDatanodeProtocolProxy( + lb.getLocations()[0], conf, 60000, false); + token = lb.getBlockToken(); + + // get block and generation stamp + final ExtendedBlock blk = new ExtendedBlock(lb.getBlock()); + originalBlock = new ExtendedBlock(blk); + originalGS = originalBlock.getGenerationStamp(); + + // test getBlockLocalPathInfo + final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token); + Assert.assertEquals(originalGS, info.getBlock().getGenerationStamp()); + } + + { // append one byte + FSDataOutputStream out = dfs.append(path); + out.write(1); + out.close(); + } + + { + // get new generation stamp + final LocatedBlock lb = cluster.getNameNode().getRpcServer() + .getBlockLocations(path.toString(), 0, 1).get(0); + final long newGS = lb.getBlock().getGenerationStamp(); + Assert.assertTrue(newGS > originalGS); + + // getBlockLocalPathInfo using the original block. + Assert.assertEquals(originalGS, originalBlock.getGenerationStamp()); + final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo( + originalBlock, token); + Assert.assertEquals(newGS, info.getBlock().getGenerationStamp()); + } + cluster.shutdown(); + } }