hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From su...@apache.org
Subject [07/50] [abbrv] hadoop git commit: Addendum patch for HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException. Contributed by Manoj Govindassamy.
Date Wed, 14 Sep 2016 02:43:45 GMT
 Addendum patch for HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException.
Contributed by Manoj Govindassamy.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0b03836
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0b03836
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0b03836

Branch: refs/heads/YARN-2915
Commit: a0b0383677c037d4492907e5f119eb0e72390faf
Parents: d4d0768
Author: Xiao Chen <xiao@apache.org>
Authored: Fri Sep 9 10:52:05 2016 -0700
Committer: Xiao Chen <xiao@apache.org>
Committed: Fri Sep 9 10:54:14 2016 -0700

----------------------------------------------------------------------
 .../fsdataset/impl/TestFsDatasetImpl.java       | 83 +++++++++++++-------
 1 file changed, 54 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b03836/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index b946803..b3f04d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -589,20 +589,21 @@ public class TestFsDatasetImpl {
     // Will write and remove on dn0.
     final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0);
     final CountDownLatch startFinalizeLatch = new CountDownLatch(1);
-    final CountDownLatch brReceivedLatch = new CountDownLatch(1);
-    final CountDownLatch volRemovedLatch = new CountDownLatch(1);
+    final CountDownLatch blockReportReceivedLatch = new CountDownLatch(1);
+    final CountDownLatch volRemoveStartedLatch = new CountDownLatch(1);
+    final CountDownLatch volRemoveCompletedLatch = new CountDownLatch(1);
     class BlockReportThread extends Thread {
       public void run() {
         // Lets wait for the volume remove process to start
         try {
-          volRemovedLatch.await();
+          volRemoveStartedLatch.await();
         } catch (Exception e) {
           LOG.info("Unexpected exception when waiting for vol removal:", e);
         }
         LOG.info("Getting block report");
         dataset.getBlockReports(eb.getBlockPoolId());
         LOG.info("Successfully received block report");
-        brReceivedLatch.countDown();
+        blockReportReceivedLatch.countDown();
       }
     }
 
@@ -623,7 +624,7 @@ public class TestFsDatasetImpl {
           }
 
           // Lets wait for the other thread finish getting block report
-          brReceivedLatch.await();
+          blockReportReceivedLatch.await();
 
           dataset.finalizeBlock(eb);
           LOG.info("FinalizeBlock finished");
@@ -633,34 +634,58 @@ public class TestFsDatasetImpl {
       }
     }
 
-    ResponderThread res = new ResponderThread();
-    res.start();
+    class VolRemoveThread extends Thread {
+      public void run() {
+        try {
+          Set<File> volumesToRemove = new HashSet<>();
+          volumesToRemove.add(StorageLocation.parse(
+              dataset.getVolume(eb).getBasePath()).getFile());
+          /**
+           * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing
+           * IllegalMonitorStateException when there is a parallel reader/writer
+           * to the volume. Remove below exception handling block after fixing
+           * HDFS-10830.
+           */
+          LOG.info("Removing volume " + volumesToRemove);
+          dataset.removeVolumes(volumesToRemove, true);
+          volRemoveCompletedLatch.countDown();
+          LOG.info("Removed volume " + volumesToRemove);
+        } catch (Exception e) {
+          LOG.info("Unexpected issue while removing volume: ", e);
+          volRemoveCompletedLatch.countDown();
+        }
+      }
+    }
+
+    // Start the volume write operation
+    ResponderThread responderThread = new ResponderThread();
+    responderThread.start();
     startFinalizeLatch.await();
 
-    // Verify if block report can be received
-    // when volume is being removed
-    final BlockReportThread brt = new BlockReportThread();
-    brt.start();
+    // Start the block report get operation
+    final BlockReportThread blockReportThread = new BlockReportThread();
+    blockReportThread.start();
 
-    Set<File> volumesToRemove = new HashSet<>();
-    volumesToRemove.add(
-        StorageLocation.parse(dataset.getVolume(eb).getBasePath()).getFile());
-    /**
-     * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing
-     * IllegalMonitorStateException when there is a parallel reader/writer
-     * to the volume. Remove below try/catch block after fixing HDFS-10830.
-     */
-    try {
-      LOG.info("Removing volume " + volumesToRemove);
-      dataset.removeVolumes(volumesToRemove, true);
-    } catch (Exception e) {
-      LOG.info("Unexpected issue while removing volume: ", e);
-    } finally {
-      volRemovedLatch.countDown();
-    }
+    // Start the volume remove operation
+    VolRemoveThread volRemoveThread = new VolRemoveThread();
+    volRemoveThread.start();
+
+    // Let volume write and remove operation be
+    // blocked for few seconds
+    Thread.sleep(2000);
+
+    // Signal block report receiver and volume writer
+    // thread to complete their operations so that vol
+    // remove can proceed
+    volRemoveStartedLatch.countDown();
+
+    // Verify if block report can be received
+    // when volume is in use and also being removed
+    blockReportReceivedLatch.await();
 
-    LOG.info("Volumes removed");
-    brReceivedLatch.await();
+    // Verify if volume can be removed safely when there
+    // are read/write operation in-progress
+    volRemoveCompletedLatch.await();
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message