Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 07097200B6B for ; Fri, 9 Sep 2016 19:54:49 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 05937160AC2; Fri, 9 Sep 2016 17:54:49 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 29A4B160AA3 for ; Fri, 9 Sep 2016 19:54:48 +0200 (CEST) Received: (qmail 64034 invoked by uid 500); 9 Sep 2016 17:54:47 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 64025 invoked by uid 99); 9 Sep 2016 17:54:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 Sep 2016 17:54:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 18E8CE01BD; Fri, 9 Sep 2016 17:54:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xiao@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: Addendum patch for HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException. Contributed by Manoj Govindassamy. Date: Fri, 9 Sep 2016 17:54:47 +0000 (UTC) archived-at: Fri, 09 Sep 2016 17:54:49 -0000 Repository: hadoop Updated Branches: refs/heads/trunk d4d076876 -> a0b038367 Addendum patch for HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException. Contributed by Manoj Govindassamy. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0b03836 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0b03836 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0b03836 Branch: refs/heads/trunk Commit: a0b0383677c037d4492907e5f119eb0e72390faf Parents: d4d0768 Author: Xiao Chen Authored: Fri Sep 9 10:52:05 2016 -0700 Committer: Xiao Chen Committed: Fri Sep 9 10:54:14 2016 -0700 ---------------------------------------------------------------------- .../fsdataset/impl/TestFsDatasetImpl.java | 83 +++++++++++++------- 1 file changed, 54 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b03836/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index b946803..b3f04d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -589,20 +589,21 @@ public class TestFsDatasetImpl { // Will write and remove on dn0. final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0); final CountDownLatch startFinalizeLatch = new CountDownLatch(1); - final CountDownLatch brReceivedLatch = new CountDownLatch(1); - final CountDownLatch volRemovedLatch = new CountDownLatch(1); + final CountDownLatch blockReportReceivedLatch = new CountDownLatch(1); + final CountDownLatch volRemoveStartedLatch = new CountDownLatch(1); + final CountDownLatch volRemoveCompletedLatch = new CountDownLatch(1); class BlockReportThread extends Thread { public void run() { // Lets wait for the volume remove process to start try { - volRemovedLatch.await(); + volRemoveStartedLatch.await(); } catch (Exception e) { LOG.info("Unexpected exception when waiting for vol removal:", e); } LOG.info("Getting block report"); dataset.getBlockReports(eb.getBlockPoolId()); LOG.info("Successfully received block report"); - brReceivedLatch.countDown(); + blockReportReceivedLatch.countDown(); } } @@ -623,7 +624,7 @@ public class TestFsDatasetImpl { } // Lets wait for the other thread finish getting block report - brReceivedLatch.await(); + blockReportReceivedLatch.await(); dataset.finalizeBlock(eb); LOG.info("FinalizeBlock finished"); @@ -633,34 +634,58 @@ public class TestFsDatasetImpl { } } - ResponderThread res = new ResponderThread(); - res.start(); + class VolRemoveThread extends Thread { + public void run() { + try { + Set volumesToRemove = new HashSet<>(); + volumesToRemove.add(StorageLocation.parse( + dataset.getVolume(eb).getBasePath()).getFile()); + /** + * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing + * IllegalMonitorStateException when there is a parallel reader/writer + * to the volume. Remove below exception handling block after fixing + * HDFS-10830. + */ + LOG.info("Removing volume " + volumesToRemove); + dataset.removeVolumes(volumesToRemove, true); + volRemoveCompletedLatch.countDown(); + LOG.info("Removed volume " + volumesToRemove); + } catch (Exception e) { + LOG.info("Unexpected issue while removing volume: ", e); + volRemoveCompletedLatch.countDown(); + } + } + } + + // Start the volume write operation + ResponderThread responderThread = new ResponderThread(); + responderThread.start(); startFinalizeLatch.await(); - // Verify if block report can be received - // when volume is being removed - final BlockReportThread brt = new BlockReportThread(); - brt.start(); + // Start the block report get operation + final BlockReportThread blockReportThread = new BlockReportThread(); + blockReportThread.start(); - Set volumesToRemove = new HashSet<>(); - volumesToRemove.add( - StorageLocation.parse(dataset.getVolume(eb).getBasePath()).getFile()); - /** - * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing - * IllegalMonitorStateException when there is a parallel reader/writer - * to the volume. Remove below try/catch block after fixing HDFS-10830. - */ - try { - LOG.info("Removing volume " + volumesToRemove); - dataset.removeVolumes(volumesToRemove, true); - } catch (Exception e) { - LOG.info("Unexpected issue while removing volume: ", e); - } finally { - volRemovedLatch.countDown(); - } + // Start the volume remove operation + VolRemoveThread volRemoveThread = new VolRemoveThread(); + volRemoveThread.start(); + + // Let volume write and remove operation be + // blocked for few seconds + Thread.sleep(2000); + + // Signal block report receiver and volume writer + // thread to complete their operations so that vol + // remove can proceed + volRemoveStartedLatch.countDown(); + + // Verify if block report can be received + // when volume is in use and also being removed + blockReportReceivedLatch.await(); - LOG.info("Volumes removed"); - brReceivedLatch.await(); + // Verify if volume can be removed safely when there + // are read/write operation in-progress + volRemoveCompletedLatch.await(); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org