From common-commits-return-97975-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Jan 28 17:40:36 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 64307180658 for ; Tue, 28 Jan 2020 18:40:36 +0100 (CET) Received: (qmail 67114 invoked by uid 500); 28 Jan 2020 17:40:35 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 67105 invoked by uid 99); 28 Jan 2020 17:40:35 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Jan 2020 17:40:35 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 123478D844; Tue, 28 Jan 2020 17:40:34 +0000 (UTC) Date: Tue, 28 Jan 2020 17:40:34 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch trunk updated: HDFS-14993. checkDiskError doesn't work during datanode startup. Contributed by Yang Yun. MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <158023323417.31196.6423244729073941283@gitbox.apache.org> From: ayushsaxena@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/trunk X-Git-Reftype: branch X-Git-Oldrev: 08251538fe2550d9dd86f9daf79994f5b8bdf7fa X-Git-Newrev: 87c198468bb6a6312bbb27b174c18822b6b9ccf8 X-Git-Rev: 87c198468bb6a6312bbb27b174c18822b6b9ccf8 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 87c1984 HDFS-14993. checkDiskError doesn't work during datanode startup. Contributed by Yang Yun. 87c1984 is described below commit 87c198468bb6a6312bbb27b174c18822b6b9ccf8 Author: Ayush Saxena AuthorDate: Tue Jan 28 22:59:26 2020 +0530 HDFS-14993. checkDiskError doesn't work during datanode startup. Contributed by Yang Yun. --- .../hadoop/hdfs/server/datanode/DataNode.java | 6 ++--- .../server/datanode/TestDataNodeVolumeFailure.java | 31 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 3b7b13d..a86f132 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1701,14 +1701,14 @@ public class DataNode extends ReconfigurableBase // the dataset, block scanners, etc. initStorage(nsInfo); - // Exclude failed disks before initializing the block pools to avoid startup - // failures. - checkDiskError(); try { data.addBlockPool(nsInfo.getBlockPoolID(), getConf()); } catch (AddBlockPoolException e) { handleAddBlockPoolError(e); } + // HDFS-14993: check disk after add the block pool info. + checkDiskError(); + blockScanner.enableBlockPoolId(bpos.getBlockPoolId()); initDirectoryScanner(getConf()); initDiskBalancer(data, getConf()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 7ad012b..c452751 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -916,4 +916,35 @@ public class TestDataNodeVolumeFailure { } } } + + /* + * Verify the failed volume can be cheched during dn startup + */ + @Test(timeout = 120000) + public void testVolumeFailureDuringStartup() throws Exception { + LOG.debug("Data dir: is " + dataDir.getPath()); + + // fail the volume + data_fail = cluster.getInstanceStorageDir(1, 0); + failedDir = MiniDFSCluster.getFinalizedDir(data_fail, + cluster.getNamesystem().getBlockPoolId()); + failedDir.setReadOnly(); + + // restart the dn + cluster.restartDataNode(1); + final DataNode dn = cluster.getDataNodes().get(1); + + // should get the failed volume during startup + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return dn.getFSDataset() !=null && + dn.getFSDataset().getVolumeFailureSummary() != null && + dn.getFSDataset().getVolumeFailureSummary(). + getFailedStorageLocations()!= null && + dn.getFSDataset().getVolumeFailureSummary(). + getFailedStorageLocations().length == 1; + } + }, 10, 30 * 1000); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org