hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shashik...@apache.org
Subject [hadoop-ozone] branch master updated: HDDS-3115. NPE seen in datanode log as ApplyTransaction failed. (#626)
Date Tue, 03 Mar 2020 04:53:46 GMT
This is an automated email from the ASF dual-hosted git repository.

shashikant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/master by this push:
     new d14db92  HDDS-3115. NPE seen in datanode log as ApplyTransaction failed. (#626)
d14db92 is described below

commit d14db928a95693399f86aa8b080b03e3a98e2d6e
Author: Siddharth <swagle@apache.org>
AuthorDate: Mon Mar 2 20:53:36 2020 -0800

    HDDS-3115. NPE seen in datanode log as ApplyTransaction failed. (#626)
---
 .../container/common/impl/HddsDispatcher.java      | 13 ++++---
 .../container/common/impl/TestHddsDispatcher.java  | 42 ++++++++++++++++++++++
 2 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 5a702d5..cef1c8f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -178,14 +178,13 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor
{
     Map<String, String> params =
         ContainerCommandRequestPBHelper.getAuditParams(msg);
 
-    Container container;
     ContainerType containerType;
     ContainerCommandResponseProto responseProto = null;
     long startTime = System.nanoTime();
     ContainerProtos.Type cmdType = msg.getCmdType();
     long containerID = msg.getContainerID();
     metrics.incContainerOpsMetrics(cmdType);
-    container = getContainer(containerID);
+    Container container = getContainer(containerID);
     boolean isWriteStage =
         (cmdType == ContainerProtos.Type.WriteChunk && dispatcherContext != null
             && dispatcherContext.getStage()
@@ -221,10 +220,10 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor
{
       // just add it to the list, and remove it from missing container set
       // as it might have been added in the list during "init".
       Preconditions.checkNotNull(container2BCSIDMap);
-      if (container2BCSIDMap.get(containerID) == null) {
-        container2BCSIDMap
-            .put(containerID, container.getBlockCommitSequenceId());
-        containerSet.getMissingContainerSet().remove(containerID);
+      if (container != null && container2BCSIDMap.get(containerID) == null) {
+        container2BCSIDMap.put(
+            containerID, container.getBlockCommitSequenceId());
+        getMissingContainerSet().remove(containerID);
       }
     }
     if (getMissingContainerSet().contains(containerID)) {
@@ -258,7 +257,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
         if (container2BCSIDMap != null) {
           // adds this container to list of containers created in the pipeline
           // with initial BCSID recorded as 0.
-          container2BCSIDMap.putIfAbsent(containerID, Long.valueOf(0));
+          container2BCSIDMap.putIfAbsent(containerID, 0L);
         }
         container = getContainer(containerID);
       }
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 5ed8b2c..ddf1f68 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.Container;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
 import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
@@ -56,6 +57,7 @@ import org.mockito.Mockito;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 import java.util.UUID;
 import java.util.function.Consumer;
@@ -166,6 +168,46 @@ public class TestHddsDispatcher {
   }
 
   @Test
+  public void testContainerNotFoundWithCommitChunk() throws IOException {
+    String testDir =
+        GenericTestUtils.getTempPath(TestHddsDispatcher.class.getSimpleName());
+    try {
+      UUID scmId = UUID.randomUUID();
+      OzoneConfiguration conf = new OzoneConfiguration();
+      conf.set(HDDS_DATANODE_DIR_KEY, testDir);
+      DatanodeDetails dd = randomDatanodeDetails();
+      HddsDispatcher hddsDispatcher = createDispatcher(dd, scmId, conf);
+      ContainerCommandRequestProto writeChunkRequest =
+          getWriteChunkRequest(dd.getUuidString(), 1L, 1L);
+
+      // send read chunk request and make sure container does not exist
+      ContainerCommandResponseProto response =
+          hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest), null);
+      Assert.assertEquals(
+          ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult());
+      DispatcherContext dispatcherContext =
+          new DispatcherContext.Builder()
+              .setContainer2BCSIDMap(Collections.emptyMap())
+              .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA)
+              .build();
+
+      GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+          .captureLogs(HddsDispatcher.LOG);
+      // send write chunk request without sending create container
+      response = hddsDispatcher.dispatch(writeChunkRequest, dispatcherContext);
+      // container should not be found
+      Assert.assertEquals(
+          ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult());
+
+      assertTrue(logCapturer.getOutput().contains(
+          "ContainerID " + writeChunkRequest.getContainerID()
+              + " does not exist"));
+    } finally {
+      FileUtils.deleteDirectory(new File(testDir));
+    }
+  }
+
+  @Test
   public void testWriteChunkWithCreateContainerFailure() throws IOException {
     String testDir = GenericTestUtils.getTempPath(
         TestHddsDispatcher.class.getSimpleName());


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message