hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject [1/2] hadoop git commit: HDDS-325. Add event watcher for delete blocks command. Contributed by Lokesh Jain.
Date Mon, 01 Oct 2018 08:24:26 GMT
Repository: hadoop
Updated Branches:
  refs/heads/ozone-0.2 c04f36db9 -> 1cc129bb7


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cc129bb/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 3c77687..1e0317f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ContainerInfo;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
 import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdfs.DFSUtil;
@@ -46,13 +47,14 @@ import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
 import org.apache.hadoop.ozone.ozShell.TestOzoneShell;
+import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
 import org.apache.hadoop.utils.MetadataStore;
 import org.junit.Assert;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
+import org.slf4j.event.Level;
 
 import java.io.File;
 import java.io.IOException;
@@ -64,8 +66,13 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import static java.lang.Math.max;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
 import static org.apache.hadoop.hdds
     .HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_DEFAULT;
 import static org.apache.hadoop.ozone
     .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
 
@@ -73,14 +80,16 @@ public class TestBlockDeletion {
   private static OzoneConfiguration conf = null;
   private static ObjectStore store;
   private static MiniOzoneCluster cluster = null;
-  private static ContainerSet dnContainerSet = null;
   private static StorageContainerManager scm = null;
   private static OzoneManager om = null;
   private static Set<Long> containerIdsWithDeletedBlocks;
+  private static long maxTransactionId = 0;
 
   @BeforeClass
   public static void init() throws Exception {
     conf = new OzoneConfiguration();
+    GenericTestUtils.setLogLevel(DeletedBlockLogImpl.LOG, Level.DEBUG);
+    GenericTestUtils.setLogLevel(SCMBlockDeletingService.LOG, Level.DEBUG);
 
     String path =
         GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName());
@@ -91,6 +100,11 @@ public class TestBlockDeletion {
         TimeUnit.MILLISECONDS);
     conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
         TimeUnit.MILLISECONDS);
+    conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200,
+        TimeUnit.MILLISECONDS);
+    conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
+    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
+        3, TimeUnit.SECONDS);
     conf.setQuietMode(false);
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(1)
@@ -98,15 +112,12 @@ public class TestBlockDeletion {
         .build();
     cluster.waitForClusterToBeReady();
     store = OzoneClientFactory.getRpcClient(conf).getObjectStore();
-    dnContainerSet = cluster.getHddsDatanodes().get(0)
-        .getDatanodeStateMachine().getContainer().getContainerSet();
     om = cluster.getOzoneManager();
     scm = cluster.getStorageContainerManager();
     containerIdsWithDeletedBlocks = new HashSet<>();
   }
 
-  @Test(timeout = 60000)
-  @Ignore("Until delete background service is fixed.")
+  @Test
   public void testBlockDeletion()
       throws IOException, InterruptedException, TimeoutException {
     String volumeName = UUID.randomUUID().toString();
@@ -151,7 +162,9 @@ public class TestBlockDeletion {
     Assert
         .assertTrue(
             OzoneTestUtils.closeContainers(omKeyLocationInfoGroupList, scm));
-    Thread.sleep(5000);
+
+    waitForDatanodeCommandRetry();
+    waitForDatanodeBlockDeletionStart();
     // The blocks should be deleted in the DN.
     Assert.assertTrue(verifyBlocksDeleted(omKeyLocationInfoGroupList));
 
@@ -166,11 +179,52 @@ public class TestBlockDeletion {
     matchContainerTransactionIds();
 
     // verify PENDING_DELETE_STATUS event is fired
-    verifyBlockDeletionEvent();
+    verifyPendingDeleteEvent();
+
+    // Verify transactions committed
+    verifyTransactionsCommitted();
+  }
+
+  private void waitForDatanodeBlockDeletionStart()
+      throws TimeoutException, InterruptedException {
+    LogCapturer logCapturer =
+        LogCapturer.captureLogs(DeleteBlocksCommandHandler.LOG);
+    logCapturer.clearOutput();
+    GenericTestUtils.waitFor(() -> logCapturer.getOutput()
+            .contains("Start to delete container block"),
+        500, 10000);
+    Thread.sleep(1000);
+  }
+
+  /**
+   * Waits for datanode command to be retried when datanode is dead.
+   */
+  private void waitForDatanodeCommandRetry()
+      throws TimeoutException, InterruptedException {
+    cluster.shutdownHddsDatanode(0);
+    LogCapturer logCapturer =
+        LogCapturer.captureLogs(RetriableDatanodeEventWatcher.LOG);
+    logCapturer.clearOutput();
+    GenericTestUtils.waitFor(() -> logCapturer.getOutput()
+            .contains("RetriableDatanodeCommand type=deleteBlocksCommand"),
+        500, 5000);
+    cluster.restartHddsDatanode(0);
+  }
+
+  private void verifyTransactionsCommitted() throws IOException {
+    DeletedBlockLogImpl deletedBlockLog =
+        (DeletedBlockLogImpl) scm.getScmBlockManager().getDeletedBlockLog();
+    for (int txnID = 1; txnID <= maxTransactionId; txnID++) {
+      Assert.assertNull(
+          deletedBlockLog.getDeletedStore().get(Longs.toByteArray(txnID)));
+    }
   }
 
-  private void verifyBlockDeletionEvent()
+  private void verifyPendingDeleteEvent()
       throws IOException, InterruptedException {
+    ContainerSet dnContainerSet =
+        cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+            .getContainer().getContainerSet();
     LogCapturer logCapturer =
         LogCapturer.captureLogs(SCMBlockDeletingService.LOG);
     // Create dummy container reports with deleteTransactionId set as 0
@@ -206,6 +260,9 @@ public class TestBlockDeletion {
   }
 
   private void matchContainerTransactionIds() throws IOException {
+    ContainerSet dnContainerSet =
+        cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+            .getContainer().getContainerSet();
     List<ContainerData> containerDataList = new ArrayList<>();
     dnContainerSet.listContainer(0, 10000, containerDataList);
     for (ContainerData containerData : containerDataList) {
@@ -213,6 +270,8 @@ public class TestBlockDeletion {
       if (containerIdsWithDeletedBlocks.contains(containerId)) {
         Assert.assertTrue(
             scm.getContainerInfo(containerId).getDeleteTransactionId() > 0);
+        maxTransactionId = max(maxTransactionId,
+            scm.getContainerInfo(containerId).getDeleteTransactionId());
       } else {
         Assert.assertEquals(
             scm.getContainerInfo(containerId).getDeleteTransactionId(), 0);
@@ -227,6 +286,9 @@ public class TestBlockDeletion {
   private boolean verifyBlocksCreated(
       List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
       throws IOException {
+    ContainerSet dnContainerSet =
+        cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+            .getContainer().getContainerSet();
     return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
       try {
         MetadataStore db = KeyUtils.getDB((KeyValueContainerData)
@@ -242,6 +304,9 @@ public class TestBlockDeletion {
   private boolean verifyBlocksDeleted(
       List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
       throws IOException {
+    ContainerSet dnContainerSet =
+        cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+            .getContainer().getContainerSet();
     return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
       try {
         MetadataStore db = KeyUtils.getDB((KeyValueContainerData)


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message