From common-commits-return-88690-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Mon Oct 1 10:24:28 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id BF7DD180660 for ; Mon, 1 Oct 2018 10:24:27 +0200 (CEST) Received: (qmail 60607 invoked by uid 500); 1 Oct 2018 08:24:26 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 60598 invoked by uid 99); 1 Oct 2018 08:24:26 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Oct 2018 08:24:26 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 40F97DFE18; Mon, 1 Oct 2018 08:24:26 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: nanda@apache.org To: common-commits@hadoop.apache.org Date: Mon, 01 Oct 2018 08:24:26 -0000 Message-Id: <65014b9529fc4405a4dfdb9291718911@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hadoop git commit: HDDS-325. Add event watcher for delete blocks command. Contributed by Lokesh Jain. Repository: hadoop Updated Branches: refs/heads/ozone-0.2 c04f36db9 -> 1cc129bb7 http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cc129bb/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java index 3c77687..1e0317f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerInfo; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl; import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdfs.DFSUtil; @@ -46,13 +47,14 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.ozShell.TestOzoneShell; +import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.utils.MetadataStore; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; +import org.slf4j.event.Level; import java.io.File; import java.io.IOException; @@ -64,8 +66,13 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static java.lang.Math.max; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL; import static org.apache.hadoop.hdds .HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; @@ -73,14 +80,16 @@ public class TestBlockDeletion { private static OzoneConfiguration conf = null; private static ObjectStore store; private static MiniOzoneCluster cluster = null; - private static ContainerSet dnContainerSet = null; private static StorageContainerManager scm = null; private static OzoneManager om = null; private static Set containerIdsWithDeletedBlocks; + private static long maxTransactionId = 0; @BeforeClass public static void init() throws Exception { conf = new OzoneConfiguration(); + GenericTestUtils.setLogLevel(DeletedBlockLogImpl.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(SCMBlockDeletingService.LOG, Level.DEBUG); String path = GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName()); @@ -91,6 +100,11 @@ public class TestBlockDeletion { TimeUnit.MILLISECONDS); conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS); + conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200, + TimeUnit.MILLISECONDS); + conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS); + conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, + 3, TimeUnit.SECONDS); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(1) @@ -98,15 +112,12 @@ public class TestBlockDeletion { .build(); cluster.waitForClusterToBeReady(); store = OzoneClientFactory.getRpcClient(conf).getObjectStore(); - dnContainerSet = cluster.getHddsDatanodes().get(0) - .getDatanodeStateMachine().getContainer().getContainerSet(); om = cluster.getOzoneManager(); scm = cluster.getStorageContainerManager(); containerIdsWithDeletedBlocks = new HashSet<>(); } - @Test(timeout = 60000) - @Ignore("Until delete background service is fixed.") + @Test public void testBlockDeletion() throws IOException, InterruptedException, TimeoutException { String volumeName = UUID.randomUUID().toString(); @@ -151,7 +162,9 @@ public class TestBlockDeletion { Assert .assertTrue( OzoneTestUtils.closeContainers(omKeyLocationInfoGroupList, scm)); - Thread.sleep(5000); + + waitForDatanodeCommandRetry(); + waitForDatanodeBlockDeletionStart(); // The blocks should be deleted in the DN. Assert.assertTrue(verifyBlocksDeleted(omKeyLocationInfoGroupList)); @@ -166,11 +179,52 @@ public class TestBlockDeletion { matchContainerTransactionIds(); // verify PENDING_DELETE_STATUS event is fired - verifyBlockDeletionEvent(); + verifyPendingDeleteEvent(); + + // Verify transactions committed + verifyTransactionsCommitted(); + } + + private void waitForDatanodeBlockDeletionStart() + throws TimeoutException, InterruptedException { + LogCapturer logCapturer = + LogCapturer.captureLogs(DeleteBlocksCommandHandler.LOG); + logCapturer.clearOutput(); + GenericTestUtils.waitFor(() -> logCapturer.getOutput() + .contains("Start to delete container block"), + 500, 10000); + Thread.sleep(1000); + } + + /** + * Waits for datanode command to be retried when datanode is dead. + */ + private void waitForDatanodeCommandRetry() + throws TimeoutException, InterruptedException { + cluster.shutdownHddsDatanode(0); + LogCapturer logCapturer = + LogCapturer.captureLogs(RetriableDatanodeEventWatcher.LOG); + logCapturer.clearOutput(); + GenericTestUtils.waitFor(() -> logCapturer.getOutput() + .contains("RetriableDatanodeCommand type=deleteBlocksCommand"), + 500, 5000); + cluster.restartHddsDatanode(0); + } + + private void verifyTransactionsCommitted() throws IOException { + DeletedBlockLogImpl deletedBlockLog = + (DeletedBlockLogImpl) scm.getScmBlockManager().getDeletedBlockLog(); + for (int txnID = 1; txnID <= maxTransactionId; txnID++) { + Assert.assertNull( + deletedBlockLog.getDeletedStore().get(Longs.toByteArray(txnID))); + } } - private void verifyBlockDeletionEvent() + private void verifyPendingDeleteEvent() throws IOException, InterruptedException { + ContainerSet dnContainerSet = + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet(); LogCapturer logCapturer = LogCapturer.captureLogs(SCMBlockDeletingService.LOG); // Create dummy container reports with deleteTransactionId set as 0 @@ -206,6 +260,9 @@ public class TestBlockDeletion { } private void matchContainerTransactionIds() throws IOException { + ContainerSet dnContainerSet = + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet(); List containerDataList = new ArrayList<>(); dnContainerSet.listContainer(0, 10000, containerDataList); for (ContainerData containerData : containerDataList) { @@ -213,6 +270,8 @@ public class TestBlockDeletion { if (containerIdsWithDeletedBlocks.contains(containerId)) { Assert.assertTrue( scm.getContainerInfo(containerId).getDeleteTransactionId() > 0); + maxTransactionId = max(maxTransactionId, + scm.getContainerInfo(containerId).getDeleteTransactionId()); } else { Assert.assertEquals( scm.getContainerInfo(containerId).getDeleteTransactionId(), 0); @@ -227,6 +286,9 @@ public class TestBlockDeletion { private boolean verifyBlocksCreated( List omKeyLocationInfoGroups) throws IOException { + ContainerSet dnContainerSet = + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet(); return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> { try { MetadataStore db = KeyUtils.getDB((KeyValueContainerData) @@ -242,6 +304,9 @@ public class TestBlockDeletion { private boolean verifyBlocksDeleted( List omKeyLocationInfoGroups) throws IOException { + ContainerSet dnContainerSet = + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet(); return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> { try { MetadataStore db = KeyUtils.getDB((KeyValueContainerData) --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org