Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 52AC710071 for ; Thu, 14 May 2015 10:25:44 +0000 (UTC) Received: (qmail 77558 invoked by uid 500); 14 May 2015 10:25:44 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 77373 invoked by uid 500); 14 May 2015 10:25:44 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77238 invoked by uid 99); 14 May 2015 10:25:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 May 2015 10:25:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8DAC5DF9A3; Thu, 14 May 2015 10:25:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Date: Thu, 14 May 2015 10:25:44 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] hadoop git commit: HDFS-8150. Make getFileChecksum fail for blocks under construction (Contributed by J.Andreina) HDFS-8150. Make getFileChecksum fail for blocks under construction (Contributed by J.Andreina) (cherry picked from commit def9136e0259e118e6fd7b656260765d28ac9ae6) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bbcffa5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bbcffa5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bbcffa5 Branch: refs/heads/branch-2 Commit: 4bbcffa510f7f93822c68204c408a1acebf431bc Parents: 085841d Author: Vinayakumar B Authored: Thu May 14 15:54:51 2015 +0530 Committer: Vinayakumar B Committed: Thu May 14 15:55:20 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 8 ++++++++ .../apache/hadoop/hdfs/TestGetFileChecksum.java | 19 +++++++++++++++++++ .../snapshot/TestSnapshotFileLength.java | 17 +++++++++++++---- 4 files changed, 43 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 17ee336..5961d34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -428,6 +428,9 @@ Release 2.8.0 - UNRELEASED HDFS-7728. Avoid updating quota usage while loading edits. (Jing Zhao via wheat9) + HDFS-8150. Make getFileChecksum fail for blocks under construction + (J.Andreina via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 216c843..60c53b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1871,6 +1871,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, if (null == blockLocations) { throw new FileNotFoundException("File does not exist: " + src); } + if (blockLocations.isUnderConstruction()) { + throw new IOException("Fail to get checksum, since file " + src + + " is under construction."); + } List locatedblocks = blockLocations.getLocatedBlocks(); final DataOutputBuffer md5out = new DataOutputBuffer(); int bytesPerCRC = -1; @@ -1890,6 +1894,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, if (null == blockLocations) { throw new FileNotFoundException("File does not exist: " + src); } + if (blockLocations.isUnderConstruction()) { + throw new IOException("Fail to get checksum, since file " + src + + " is under construction."); + } locatedblocks = blockLocations.getLocatedBlocks(); refetchBlocks = false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java index 0e56ba7..814261f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java @@ -17,7 +17,12 @@ */ package org.apache.hadoop.hdfs; +import static org.junit.Assert.fail; + +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.Path; import org.junit.After; @@ -68,6 +73,20 @@ public class TestGetFileChecksum { } @Test + public void testGetFileChecksumForBlocksUnderConstruction() { + try { + FSDataOutputStream file = dfs.create(new Path("/testFile")); + file.write("Performance Testing".getBytes()); + dfs.getFileChecksum(new Path("/testFile")); + fail("getFileChecksum should fail for files " + + "with blocks under construction"); + } catch (IOException ie) { + Assert.assertTrue(ie.getMessage().contains( + "Fail to get checksum, since file /testFile " + + "is under construction.")); + } + } + @Test public void testGetFileChecksum() throws Exception { testGetFileChecksum(new Path("/foo"), BLOCKSIZE / 4); testGetFileChecksum(new Path("/bar"), BLOCKSIZE / 4 - 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java index d53140f..90e499d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.PrintStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -32,6 +33,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -111,8 +114,16 @@ public class TestSnapshotFileLength { // Append to the file. FSDataOutputStream out = hdfs.append(file1); // Nothing has been appended yet. All checksums should still be equal. - assertThat("file and snapshot checksums (open for append) are not equal", - hdfs.getFileChecksum(file1), is(snapChksum1)); + // HDFS-8150:Fetching checksum for file under construction should fail + try { + hdfs.getFileChecksum(file1); + fail("getFileChecksum should fail for files " + + "with blocks under construction"); + } catch (IOException ie) { + assertTrue(ie.getMessage().contains( + "Fail to get checksum, since file " + file1 + + " is under construction.")); + } assertThat("snapshot checksum (post-open for append) has changed", hdfs.getFileChecksum(file1snap1), is(snapChksum1)); try { @@ -122,8 +133,6 @@ public class TestSnapshotFileLength { assertThat("Wrong data size in snapshot.", dataFromSnapshot.length, is(origLen)); // Verify that checksum didn't change - assertThat("snapshot file checksum (pre-close) has changed", - hdfs.getFileChecksum(file1), is(snapChksum1)); assertThat("snapshot checksum (post-append) has changed", hdfs.getFileChecksum(file1snap1), is(snapChksum1)); } finally {