From ozone-commits-return-815-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Mar 3 04:53:48 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 4264218062C for ; Tue, 3 Mar 2020 05:53:48 +0100 (CET) Received: (qmail 37597 invoked by uid 500); 3 Mar 2020 04:53:47 -0000 Mailing-List: contact ozone-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ozone-commits@hadoop.apache.org Delivered-To: mailing list ozone-commits@hadoop.apache.org Received: (qmail 37585 invoked by uid 99); 3 Mar 2020 04:53:47 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Mar 2020 04:53:47 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 26F0D8DACA; Tue, 3 Mar 2020 04:53:46 +0000 (UTC) Date: Tue, 03 Mar 2020 04:53:46 +0000 To: "ozone-commits@hadoop.apache.org" Subject: [hadoop-ozone] branch master updated: HDDS-3115. NPE seen in datanode log as ApplyTransaction failed. (#626) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <158321122681.23772.13530462178728856688@gitbox.apache.org> From: shashikant@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop-ozone X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 9a16a2ae7eeb4f7a74f76312c7aa18cd36070574 X-Git-Newrev: d14db928a95693399f86aa8b080b03e3a98e2d6e X-Git-Rev: d14db928a95693399f86aa8b080b03e3a98e2d6e X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. shashikant pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git The following commit(s) were added to refs/heads/master by this push: new d14db92 HDDS-3115. NPE seen in datanode log as ApplyTransaction failed. (#626) d14db92 is described below commit d14db928a95693399f86aa8b080b03e3a98e2d6e Author: Siddharth AuthorDate: Mon Mar 2 20:53:36 2020 -0800 HDDS-3115. NPE seen in datanode log as ApplyTransaction failed. (#626) --- .../container/common/impl/HddsDispatcher.java | 13 ++++--- .../container/common/impl/TestHddsDispatcher.java | 42 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 5a702d5..cef1c8f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -178,14 +178,13 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { Map params = ContainerCommandRequestPBHelper.getAuditParams(msg); - Container container; ContainerType containerType; ContainerCommandResponseProto responseProto = null; long startTime = System.nanoTime(); ContainerProtos.Type cmdType = msg.getCmdType(); long containerID = msg.getContainerID(); metrics.incContainerOpsMetrics(cmdType); - container = getContainer(containerID); + Container container = getContainer(containerID); boolean isWriteStage = (cmdType == ContainerProtos.Type.WriteChunk && dispatcherContext != null && dispatcherContext.getStage() @@ -221,10 +220,10 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { // just add it to the list, and remove it from missing container set // as it might have been added in the list during "init". Preconditions.checkNotNull(container2BCSIDMap); - if (container2BCSIDMap.get(containerID) == null) { - container2BCSIDMap - .put(containerID, container.getBlockCommitSequenceId()); - containerSet.getMissingContainerSet().remove(containerID); + if (container != null && container2BCSIDMap.get(containerID) == null) { + container2BCSIDMap.put( + containerID, container.getBlockCommitSequenceId()); + getMissingContainerSet().remove(containerID); } } if (getMissingContainerSet().contains(containerID)) { @@ -258,7 +257,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { if (container2BCSIDMap != null) { // adds this container to list of containers created in the pipeline // with initial BCSID recorded as 0. - container2BCSIDMap.putIfAbsent(containerID, Long.valueOf(0)); + container2BCSIDMap.putIfAbsent(containerID, 0L); } container = getContainer(containerID); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java index 5ed8b2c..ddf1f68 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext; import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; @@ -56,6 +57,7 @@ import org.mockito.Mockito; import java.io.File; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.UUID; import java.util.function.Consumer; @@ -166,6 +168,46 @@ public class TestHddsDispatcher { } @Test + public void testContainerNotFoundWithCommitChunk() throws IOException { + String testDir = + GenericTestUtils.getTempPath(TestHddsDispatcher.class.getSimpleName()); + try { + UUID scmId = UUID.randomUUID(); + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(HDDS_DATANODE_DIR_KEY, testDir); + DatanodeDetails dd = randomDatanodeDetails(); + HddsDispatcher hddsDispatcher = createDispatcher(dd, scmId, conf); + ContainerCommandRequestProto writeChunkRequest = + getWriteChunkRequest(dd.getUuidString(), 1L, 1L); + + // send read chunk request and make sure container does not exist + ContainerCommandResponseProto response = + hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest), null); + Assert.assertEquals( + ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult()); + DispatcherContext dispatcherContext = + new DispatcherContext.Builder() + .setContainer2BCSIDMap(Collections.emptyMap()) + .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA) + .build(); + + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer + .captureLogs(HddsDispatcher.LOG); + // send write chunk request without sending create container + response = hddsDispatcher.dispatch(writeChunkRequest, dispatcherContext); + // container should not be found + Assert.assertEquals( + ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult()); + + assertTrue(logCapturer.getOutput().contains( + "ContainerID " + writeChunkRequest.getContainerID() + + " does not exist")); + } finally { + FileUtils.deleteDirectory(new File(testDir)); + } + } + + @Test public void testWriteChunkWithCreateContainerFailure() throws IOException { String testDir = GenericTestUtils.getTempPath( TestHddsDispatcher.class.getSimpleName()); --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: ozone-commits-help@hadoop.apache.org