hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject [hadoop-ozone] branch master updated: HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE
Date Mon, 11 Nov 2019 18:14:06 GMT
This is an automated email from the ASF dual-hosted git repository.

elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/master by this push:
     new a6f80c0  HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE
a6f80c0 is described below

commit a6f80c096b5320f50b6e9e9b4ba5f7c7e3544385
Author: Márton Elek <elek@apache.org>
AuthorDate: Mon Nov 11 19:09:55 2019 +0100

    HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE
    
    Closes #60
---
 .../container/common/impl/HddsDispatcher.java      |  5 ++
 .../ozone/container/common/interfaces/Handler.java | 60 ++++++++--------
 .../ozone/container/keyvalue/KeyValueHandler.java  | 12 ++--
 .../ozone/container/ozoneimpl/OzoneContainer.java  | 59 ++++++++++------
 .../container/common/impl/TestHddsDispatcher.java  | 16 +++--
 .../container/common/interfaces/TestHandler.java   |  8 ++-
 .../container/keyvalue/TestKeyValueHandler.java    | 10 ++-
 .../TestKeyValueHandlerWithUnhealthyContainer.java |  7 +-
 .../container/metrics/TestContainerMetrics.java    |  9 ++-
 .../container/server/TestContainerServer.java      |  7 +-
 .../server/TestSecureContainerServer.java          |  7 +-
 .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 82 ++++++++++++----------
 .../org/apache/hadoop/ozone/genesis/Genesis.java   |  3 +-
 13 files changed, 169 insertions(+), 116 deletions(-)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 7f46707..1feb500 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -321,6 +321,11 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
         // Once container is marked unhealthy, all the subsequent write
         // transactions will fail with UNHEALTHY_CONTAINER exception.
 
+        if (container == null) {
+          throw new NullPointerException(
+              "Error on creating containers " + result + " " + responseProto
+                  .getMessage());
+        }
         // For container to be moved to unhealthy state here, the container can
         // only be in open or closing state.
         State containerState = container.getContainerData().getState();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 8c3b981..c623e14 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -18,25 +18,19 @@
 
 package org.apache.hadoop.ozone.container.common.interfaces;
 
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.function.Consumer;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerType;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
@@ -54,41 +48,46 @@ public abstract class Handler {
   protected final VolumeSet volumeSet;
   protected String scmID;
   protected final ContainerMetrics metrics;
+  protected String datanodeId;
+  private Consumer<ContainerReplicaProto> icrSender;
 
-  private final StateContext context;
-  private final DatanodeDetails datanodeDetails;
-
-  protected Handler(Configuration config, StateContext context,
+  protected Handler(Configuration config, String datanodeId,
       ContainerSet contSet, VolumeSet volumeSet,
-      ContainerMetrics containerMetrics) {
+      ContainerMetrics containerMetrics,
+      Consumer<ContainerReplicaProto> icrSender) {
     this.conf = config;
-    this.context = context;
     this.containerSet = contSet;
     this.volumeSet = volumeSet;
     this.metrics = containerMetrics;
-    this.datanodeDetails = context.getParent().getDatanodeDetails();
+    this.datanodeId = datanodeId;
+    this.icrSender = icrSender;
   }
 
   public static Handler getHandlerForContainerType(
       final ContainerType containerType, final Configuration config,
-      final StateContext context, final ContainerSet contSet,
-      final VolumeSet volumeSet, final ContainerMetrics metrics) {
+      final String datanodeId, final ContainerSet contSet,
+      final VolumeSet volumeSet, final ContainerMetrics metrics,
+      Consumer<ContainerReplicaProto> icrSender) {
     switch (containerType) {
     case KeyValueContainer:
-      return new KeyValueHandler(config, context, contSet, volumeSet, metrics);
+      return new KeyValueHandler(config,
+          datanodeId, contSet, volumeSet, metrics,
+          icrSender);
     default:
       throw new IllegalArgumentException("Handler for ContainerType: " +
-        containerType + "doesn't exist.");
+          containerType + "doesn't exist.");
     }
   }
 
   /**
    * Returns the Id of this datanode.
+   *
    * @return datanode Id
    */
-  protected DatanodeDetails getDatanodeDetails() {
-    return datanodeDetails;
+  protected String getDatanodeId() {
+    return datanodeId;
   }
+
   /**
    * This should be called whenever there is state change. It will trigger
    * an ICR to SCM.
@@ -97,12 +96,8 @@ public abstract class Handler {
    */
   protected void sendICR(final Container container)
       throws StorageContainerException {
-    IncrementalContainerReportProto icr = IncrementalContainerReportProto
-        .newBuilder()
-        .addReport(container.getContainerReport())
-        .build();
-    context.addReport(icr);
-    context.getParent().triggerHeartbeat();
+    ContainerReplicaProto containerReport = container.getContainerReport();
+    icrSender.accept(containerReport);
   }
 
   public abstract ContainerCommandResponseProto handle(
@@ -175,8 +170,9 @@ public abstract class Handler {
    * Deletes the given container.
    *
    * @param container container to be deleted
-   * @param force if this is set to true, we delete container without checking
-   * state of the container.
+   * @param force     if this is set to true, we delete container without
+   *                  checking
+   *                  state of the container.
    * @throws IOException
    */
   public abstract void deleteContainer(Container container, boolean force)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 995b913..5da0b8f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -27,6 +27,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
 import java.util.function.Function;
 
 import org.apache.hadoop.conf.Configuration;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .PutSmallFileRequestProto;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
 import org.apache.hadoop.hdds.scm.ByteStringConversion;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.common.helpers
@@ -60,7 +62,6 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.ozone.container.common.transport.server.ratis
     .DispatcherContext;
 import org.apache.hadoop.ozone.container.common.transport.server.ratis
@@ -109,9 +110,10 @@ public class KeyValueHandler extends Handler {
   private final AutoCloseableLock containerCreationLock;
   private final boolean doSyncWrite;
 
-  public KeyValueHandler(Configuration config, StateContext context,
-      ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics) {
-    super(config, context, contSet, volSet, metrics);
+  public KeyValueHandler(Configuration config, String datanodeId,
+      ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics,
+      Consumer<ContainerReplicaProto> icrSender) {
+    super(config, datanodeId, contSet, volSet, metrics, icrSender);
     containerType = ContainerType.KeyValueContainer;
     blockManager = new BlockManagerImpl(config);
     doSyncWrite =
@@ -220,7 +222,7 @@ public class KeyValueHandler extends Handler {
 
     KeyValueContainerData newContainerData = new KeyValueContainerData(
         containerID, maxContainerSize, request.getPipelineID(),
-        getDatanodeDetails().getUuidString());
+        getDatanodeId());
     // TODO: Add support to add metadataList to ContainerData. Add metadata
     // to container during creation.
     KeyValueContainer newContainer = new KeyValueContainer(
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 6bce22c..328cd91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -18,16 +18,21 @@
 
 package org.apache.hadoop.ozone.container.ozoneimpl;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto
-    .ContainerProtos.ContainerType;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
-        .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
 import org.apache.hadoop.hdds.security.token.BlockTokenVerifier;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
@@ -42,24 +47,20 @@ import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSp
 import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-
 import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
 import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
-import org.apache.hadoop.ozone.container.replication
-    .OnDemandContainerReplicationSource;
+import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
 import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.*;
-
 /**
  * Ozone main class sets up the network servers and initializes the container
  * layer.
@@ -100,10 +101,22 @@ public class OzoneContainer {
     buildContainerSet();
     final ContainerMetrics metrics = ContainerMetrics.create(conf);
     this.handlers = Maps.newHashMap();
+
+    Consumer<ContainerReplicaProto> icrSender = containerReplicaProto -> {
+      IncrementalContainerReportProto icr = IncrementalContainerReportProto
+          .newBuilder()
+          .addReport(containerReplicaProto)
+          .build();
+      context.addReport(icr);
+      context.getParent().triggerHeartbeat();
+    };
+
     for (ContainerType containerType : ContainerType.values()) {
       handlers.put(containerType,
           Handler.getHandlerForContainerType(
-              containerType, conf, context, containerSet, volumeSet, metrics));
+              containerType, conf,
+              context.getParent().getDatanodeDetails().getUuidString(),
+              containerSet, volumeSet, metrics, icrSender));
     }
 
     SecurityConfig secConf = new SecurityConfig(conf);
@@ -169,7 +182,6 @@ public class OzoneContainer {
 
   }
 
-
   /**
    * Start background daemon thread for performing container integrity checks.
    */
@@ -240,13 +252,14 @@ public class OzoneContainer {
     ContainerMetrics.remove();
   }
 
-
   @VisibleForTesting
   public ContainerSet getContainerSet() {
     return containerSet;
   }
+
   /**
    * Returns container report.
+   *
    * @return - container report.
    */
 
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index e9e8da8..9db11d0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .WriteChunkRequestProto;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
 import org.apache.hadoop.ozone.common.Checksum;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
@@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.hadoop.test.GenericTestUtils;
+
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.junit.Assert;
 import org.junit.Test;
@@ -57,6 +59,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Map;
 import java.util.UUID;
+import java.util.function.Consumer;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
@@ -69,6 +72,9 @@ import static org.mockito.Mockito.verify;
  */
 public class TestHddsDispatcher {
 
+  public static final Consumer<ContainerReplicaProto> NO_OP_ICR_SENDER =
+      c -> {};
+
   @Test
   public void testContainerCloseActionWhenFull() throws IOException {
     String testDir = GenericTestUtils.getTempPath(
@@ -98,8 +104,9 @@ public class TestHddsDispatcher {
       Map<ContainerType, Handler> handlers = Maps.newHashMap();
       for (ContainerType containerType : ContainerType.values()) {
         handlers.put(containerType,
-            Handler.getHandlerForContainerType(containerType, conf, context,
-                containerSet, volumeSet, metrics));
+            Handler.getHandlerForContainerType(containerType, conf,
+                context.getParent().getDatanodeDetails().getUuidString(),
+                containerSet, volumeSet, metrics, NO_OP_ICR_SENDER));
       }
       HddsDispatcher hddsDispatcher = new HddsDispatcher(
           conf, containerSet, volumeSet, handlers, context, metrics, null);
@@ -214,8 +221,9 @@ public class TestHddsDispatcher {
     Map<ContainerType, Handler> handlers = Maps.newHashMap();
     for (ContainerType containerType : ContainerType.values()) {
       handlers.put(containerType,
-          Handler.getHandlerForContainerType(containerType, conf, context,
-              containerSet, volumeSet, metrics));
+          Handler.getHandlerForContainerType(containerType, conf,
+              context.getParent().getDatanodeDetails().getUuidString(),
+              containerSet, volumeSet, metrics, NO_OP_ICR_SENDER));
     }
 
     HddsDispatcher hddsDispatcher = new HddsDispatcher(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
index cb40f95..086b0fe 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@ -19,16 +19,19 @@
 package org.apache.hadoop.ozone.container.common.interfaces;
 
 import com.google.common.collect.Maps;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -69,7 +72,10 @@ public class TestHandler {
         ContainerProtos.ContainerType.values()) {
       handlers.put(containerType,
           Handler.getHandlerForContainerType(
-              containerType, conf, context, containerSet, volumeSet, metrics));
+              containerType, conf,
+              context.getParent().getDatanodeDetails().getUuidString(),
+              containerSet, volumeSet, metrics,
+              TestHddsDispatcher.NO_OP_ICR_SENDER));
     }
     this.dispatcher = new HddsDispatcher(
         conf, containerSet, volumeSet, handlers, null, metrics, null);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 2c71fef..05b92cc 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -244,8 +244,10 @@ public class TestKeyValueHandler {
       Mockito.when(stateMachine.getDatanodeDetails())
           .thenReturn(datanodeDetails);
       Mockito.when(context.getParent()).thenReturn(stateMachine);
-      KeyValueHandler keyValueHandler = new KeyValueHandler(conf, context, cset,
-          volumeSet, metrics);
+      KeyValueHandler keyValueHandler = new KeyValueHandler(conf,
+          context.getParent().getDatanodeDetails().getUuidString(), cset,
+          volumeSet, metrics, c -> {
+      });
       assertEquals("org.apache.hadoop.ozone.container.common" +
           ".volume.RoundRobinVolumeChoosingPolicy",
           keyValueHandler.getVolumeChoosingPolicyForTesting()
@@ -255,7 +257,9 @@ public class TestKeyValueHandler {
       conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
           "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");
       try {
-        new KeyValueHandler(conf, context, cset, volumeSet, metrics);
+        new KeyValueHandler(conf,
+            context.getParent().getDatanodeDetails().getUuidString(),
+            cset, volumeSet, metrics, c->{});
       } catch (RuntimeException ex) {
         GenericTestUtils.assertExceptionContains("class org.apache.hadoop" +
             ".ozone.container.common.impl.HddsDispatcher not org.apache" +
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
index e3ae56a..26603b6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
@@ -24,9 +24,11 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -41,7 +43,6 @@ import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-
 /**
  * Test that KeyValueHandler fails certain operations when the
  * container is unhealthy.
@@ -147,10 +148,10 @@ public class TestKeyValueHandlerWithUnhealthyContainer {
 
     return new KeyValueHandler(
         new OzoneConfiguration(),
-        context,
+        stateMachine.getDatanodeDetails().getUuidString(),
         mock(ContainerSet.class),
         mock(VolumeSet.class),
-        mock(ContainerMetrics.class));
+        mock(ContainerMetrics.class), TestHddsDispatcher.NO_OP_ICR_SENDER);
   }
 
   private KeyValueContainer getMockUnhealthyContainer() {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
index 1d30653..0598f15 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
 
 import com.google.common.collect.Maps;
+
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -51,6 +53,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
 import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
 import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
 import org.apache.hadoop.test.GenericTestUtils;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -104,8 +107,10 @@ public class TestContainerMetrics {
       for (ContainerProtos.ContainerType containerType :
           ContainerProtos.ContainerType.values()) {
         handlers.put(containerType,
-            Handler.getHandlerForContainerType(containerType, conf, context,
-                containerSet, volumeSet, metrics));
+            Handler.getHandlerForContainerType(containerType, conf,
+                context.getParent().getDatanodeDetails().getUuidString(),
+                containerSet, volumeSet, metrics,
+                TestHddsDispatcher.NO_OP_ICR_SENDER));
       }
       HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet,
           volumeSet, handlers, context, metrics, null);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index d71c0e7..e382734 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClie
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -224,8 +225,10 @@ public class TestContainerServer {
       for (ContainerProtos.ContainerType containerType :
           ContainerProtos.ContainerType.values()) {
         handlers.put(containerType,
-            Handler.getHandlerForContainerType(containerType, conf, context,
-                containerSet, volumeSet, metrics));
+            Handler.getHandlerForContainerType(containerType, conf,
+                context.getParent().getDatanodeDetails().getUuidString(),
+                containerSet, volumeSet, metrics,
+                TestHddsDispatcher.NO_OP_ICR_SENDER));
       }
       HddsDispatcher dispatcher = new HddsDispatcher(
           conf, containerSet, volumeSet, handlers, context, metrics, null);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index 2253fe2..33f93e5 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
@@ -155,8 +156,10 @@ public class TestSecureContainerServer {
     for (ContainerProtos.ContainerType containerType :
         ContainerProtos.ContainerType.values()) {
       handlers.put(containerType,
-          Handler.getHandlerForContainerType(containerType, conf, context,
-              containerSet, volumeSet, metrics));
+          Handler.getHandlerForContainerType(containerType, conf,
+              dd.getUuid().toString(),
+              containerSet, volumeSet, metrics,
+              TestHddsDispatcher.NO_OP_ICR_SENDER));
     }
     HddsDispatcher hddsDispatcher = new HddsDispatcher(
         conf, containerSet, volumeSet, handlers, context, metrics,
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 28da134..1d087f9 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -17,25 +17,42 @@
  */
 package org.apache.hadoop.ozone.genesis;
 
-import com.google.common.collect.Maps;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumData;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.statemachine
-    .DatanodeStateMachine.DatanodeStates;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+
+import com.google.common.collect.Maps;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Scope;
@@ -43,28 +60,6 @@ import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .PutBlockRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .GetBlockRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ReadChunkRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .WriteChunkRequestProto;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-
 /**
  * Benchmarks DatanodeDispatcher class.
  */
@@ -83,6 +78,7 @@ public class BenchMarkDatanodeDispatcher {
   private static final int INIT_CONTAINERS = 100;
   private static final int INIT_KEYS = 50;
   private static final int INIT_CHUNKS = 100;
+  public static final int CHUNK_SIZE = 1048576;
 
   private List<Long> containers;
   private List<Long> keys;
@@ -94,8 +90,8 @@ public class BenchMarkDatanodeDispatcher {
     datanodeUuid = UUID.randomUUID().toString();
 
     // 1 MB of data
-    data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(1048576));
-    random  = new Random();
+    data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(CHUNK_SIZE));
+    random = new Random();
     Configuration conf = new OzoneConfiguration();
     baseDir = System.getProperty("java.io.tmpdir") + File.separator +
         datanodeUuid;
@@ -103,6 +99,9 @@ public class BenchMarkDatanodeDispatcher {
     // data directory
     conf.set("dfs.datanode.data.dir", baseDir + File.separator + "data");
 
+    //We need 100 * container size minimum space
+    conf.set("ozone.scm.container.size", "10MB");
+
     ContainerSet containerSet = new ContainerSet();
     volumeSet = new VolumeSet(datanodeUuid, conf);
     StateContext context = new StateContext(
@@ -111,9 +110,12 @@ public class BenchMarkDatanodeDispatcher {
     Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
     for (ContainerProtos.ContainerType containerType :
         ContainerProtos.ContainerType.values()) {
-      handlers.put(containerType,
-          Handler.getHandlerForContainerType(
-              containerType, conf, context, containerSet, volumeSet, metrics));
+      Handler handler = Handler.getHandlerForContainerType(
+          containerType, conf, "datanodeid",
+          containerSet, volumeSet, metrics,
+          c -> {});
+      handler.setScmID("scm");
+      handlers.put(containerType, handler);
     }
     dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers,
         context, metrics, null);
@@ -217,11 +219,16 @@ public class BenchMarkDatanodeDispatcher {
   private ContainerProtos.ChunkInfo getChunkInfo(
       BlockID blockID, String chunkName) {
     ContainerProtos.ChunkInfo.Builder builder =
-        ContainerProtos.ChunkInfo.newBuilder()
+        ChunkInfo.newBuilder()
             .setChunkName(
                 DigestUtils.md5Hex(chunkName)
                     + "_stream_" + blockID.getContainerID() + "_block_"
                     + blockID.getLocalID())
+            .setChecksumData(
+                ChecksumData.newBuilder()
+                    .setBytesPerChecksum(4)
+                    .setType(ChecksumType.CRC32)
+                    .build())
             .setOffset(0).setLen(data.size());
     return builder.build();
   }
@@ -245,7 +252,7 @@ public class BenchMarkDatanodeDispatcher {
   private ContainerCommandRequestProto getGetBlockCommand(BlockID blockID) {
     GetBlockRequestProto.Builder readBlockRequest =
         GetBlockRequestProto.newBuilder()
-        .setBlockID(blockID.getDatanodeBlockIDProtobuf());
+            .setBlockID(blockID.getDatanodeBlockIDProtobuf());
     ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
         .newBuilder()
         .setCmdType(ContainerProtos.Type.GetBlock)
@@ -274,7 +281,6 @@ public class BenchMarkDatanodeDispatcher {
     bmdd.containerCount.getAndIncrement();
   }
 
-
   @Benchmark
   public void writeChunk(BenchMarkDatanodeDispatcher bmdd) {
     bmdd.dispatcher.dispatch(getWriteChunkCommand(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
index 2de9b0f..4361666 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
@@ -41,7 +41,8 @@ public final class Genesis {
 
   // After adding benchmark in genesis package add the benchmark name in the
   // description for this option.
-  @Option(names = "-benchmark", split = ",", description =
+  @Option(names = {"-b", "-benchmark", "--benchmark"},
+      split = ",", description =
       "Option used for specifying benchmarks to run.\n"
           + "Ex. ozone genesis -benchmark BenchMarkContainerStateMap,"
           + "BenchMarkOMKeyAllocation.\n"


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message