Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ACB3018C0E for ; Fri, 8 May 2015 11:48:32 +0000 (UTC) Received: (qmail 24747 invoked by uid 500); 8 May 2015 11:48:32 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 24676 invoked by uid 500); 8 May 2015 11:48:32 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 24667 invoked by uid 99); 8 May 2015 11:48:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 08 May 2015 11:48:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 452C2E022E; Fri, 8 May 2015 11:48:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Message-Id: <92616dec21684ac8a809fa380cc1f748@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-3384. DataStreamer thread should be closed immediatly when failed to setup a PipelineForAppendOrRecovery (Contributed by Uma Maheswara Rao G) Date: Fri, 8 May 2015 11:48:32 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 1ffb7fa42 -> c648317a6 HDFS-3384. DataStreamer thread should be closed immediatly when failed to setup a PipelineForAppendOrRecovery (Contributed by Uma Maheswara Rao G) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c648317a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c648317a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c648317a Branch: refs/heads/trunk Commit: c648317a68891e1c900f04b7a9c98ba40c5faddb Parents: 1ffb7fa Author: Vinayakumar B Authored: Fri May 8 17:18:14 2015 +0530 Committer: Vinayakumar B Committed: Fri May 8 17:18:14 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DataStreamer.java | 4 ++++ .../org/apache/hadoop/hdfs/TestFileAppend.java | 22 ++++++++++++++++++++ 3 files changed, 29 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c648317a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c9e5b87..4decfd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -690,6 +690,9 @@ Release 2.8.0 - UNRELEASED HDFS-6576. Datanode log is generating at root directory in security mode (surendra singh lilhore via vinayakumarb) + HDFS-3384. DataStreamer thread should be closed immediatly when failed to + setup a PipelineForAppendOrRecovery (Uma Maheswara Rao G via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c648317a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 96bf212..697ee11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -460,6 +460,9 @@ class DataStreamer extends Daemon { LOG.debug("Append to block " + block); } setupPipelineForAppendOrRecovery(); + if (true == streamerClosed) { + continue; + } initDataStreaming(); } @@ -571,6 +574,7 @@ class DataStreamer extends Daemon { } } lastException.set(e); + assert !(e instanceof NullPointerException); hasError = true; if (errorIndex == -1 && restartingNodeIndex.get() == -1) { // Not a datanode issue http://git-wip-us.apache.org/repos/asf/hadoop/blob/c648317a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java index 6a7c3ea..402c944 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java @@ -602,4 +602,26 @@ public class TestFileAppend{ cluster.shutdown(); } } + + @Test(timeout = 10000) + public void testAppendCorruptedBlock() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); + conf.setInt("dfs.min.replication", 1); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .build(); + try { + DistributedFileSystem fs = cluster.getFileSystem(); + Path fileName = new Path("/appendCorruptBlock"); + DFSTestUtil.createFile(fs, fileName, 512, (short) 1, 0); + DFSTestUtil.waitReplication(fs, fileName, (short) 1); + Assert.assertTrue("File not created", fs.exists(fileName)); + ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName); + cluster.corruptBlockOnDataNodes(block); + DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock"); + } finally { + cluster.shutdown(); + } + } }