hadoop-common-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [hadoop] nandakumar131 commented on a change in pull request #662: HDDS-1207. Refactor Container Report Processing logic and plugin new Replication Manager.
Date Wed, 03 Apr 2019 08:51:47 GMT
nandakumar131 commented on a change in pull request #662: HDDS-1207. Refactor Container Report
Processing logic and plugin new Replication Manager.
URL: https://github.com/apache/hadoop/pull/662#discussion_r271639382
 
 

 ##########
 File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
 ##########
 @@ -146,68 +111,89 @@ public void onMessage(final ContainerReportFromDatanode reportFromDatanode,
 
   }
 
+  /**
+   * Processes the ContainerReport.
+   *
+   * @param datanodeDetails Datanode from which this report was received
+   * @param replicas list of ContainerReplicaProto
+   */
   private void processContainerReplicas(final DatanodeDetails datanodeDetails,
-      final List<ContainerReplicaProto> replicas,
-      final EventPublisher publisher) {
-    final PendingDeleteStatusList pendingDeleteStatusList =
-        new PendingDeleteStatusList(datanodeDetails);
+      final List<ContainerReplicaProto> replicas) {
     for (ContainerReplicaProto replicaProto : replicas) {
       try {
-        final ContainerID containerID = ContainerID.valueof(
-            replicaProto.getContainerID());
-
-        ReportHandlerHelper.processContainerReplica(containerManager,
-            containerID, replicaProto, datanodeDetails, publisher, LOG);
-
-        final ContainerInfo containerInfo = containerManager
-            .getContainer(containerID);
-
-        if (containerInfo.getDeleteTransactionId() >
-            replicaProto.getDeleteTransactionId()) {
-          pendingDeleteStatusList
-              .addPendingDeleteStatus(replicaProto.getDeleteTransactionId(),
-                  containerInfo.getDeleteTransactionId(),
-                  containerInfo.getContainerID());
-        }
+        processContainerReplica(datanodeDetails, replicaProto);
       } catch (ContainerNotFoundException e) {
-        LOG.error("Received container report for an unknown container {} from"
-                + " datanode {} {}", replicaProto.getContainerID(),
+        LOG.error("Received container report for an unknown container" +
+                " {} from datanode {}.", replicaProto.getContainerID(),
             datanodeDetails, e);
       } catch (IOException e) {
-        LOG.error("Exception while processing container report for container"
-                + " {} from datanode {} {}", replicaProto.getContainerID(),
+        LOG.error("Exception while processing container report for container" +
+                " {} from datanode {}.", replicaProto.getContainerID(),
             datanodeDetails, e);
       }
     }
-    if (pendingDeleteStatusList.getNumPendingDeletes() > 0) {
-      publisher.fireEvent(SCMEvents.PENDING_DELETE_STATUS,
-          pendingDeleteStatusList);
-    }
   }
 
-  private void checkReplicationState(ContainerID containerID,
-      EventPublisher publisher) {
-    try {
-      ContainerInfo container = containerManager.getContainer(containerID);
-      replicateIfNeeded(container, publisher);
-    } catch (ContainerNotFoundException ex) {
-      LOG.warn("Container is missing from containerStateManager. Can't request "
-          + "replication. {} {}", containerID, ex);
+  /**
+   * Process the missing replica on the given datanode.
+   *
+   * @param datanodeDetails DatanodeDetails
+   * @param missingReplicas ContainerID which are missing on the given datanode
+   */
+  private void processMissingReplicas(final DatanodeDetails datanodeDetails,
+                                      final Set<ContainerID> missingReplicas) {
+    for (ContainerID id : missingReplicas) {
+      try {
+        containerManager.getContainerReplicas(id).stream()
+            .filter(replica -> replica.getDatanodeDetails()
+                .equals(datanodeDetails)).findFirst()
+            .ifPresent(replica -> {
+              try {
+                containerManager.removeContainerReplica(id, replica);
+              } catch (ContainerNotFoundException |
+                  ContainerReplicaNotFoundException ignored) {
+                // This should not happen, but even if it happens, not an issue
+              }
+            });
+      } catch (ContainerNotFoundException e) {
+        LOG.warn("Cannot remove container replica, container {} not found.",
+            id, e);
+      }
     }
-
   }
 
-  private void replicateIfNeeded(ContainerInfo container,
-      EventPublisher publisher) throws ContainerNotFoundException {
-    if (!container.isOpen() && replicationStatus.isReplicationEnabled()) {
-      final int existingReplicas = containerManager
-          .getContainerReplicas(container.containerID()).size();
-      final int expectedReplicas = container.getReplicationFactor().getNumber();
-      if (existingReplicas != expectedReplicas) {
-        publisher.fireEvent(SCMEvents.REPLICATE_CONTAINER,
-            new ReplicationRequest(container.getContainerID(),
-                existingReplicas, expectedReplicas));
+  /**
+   * Updates the Delete Transaction Id for the given datanode.
+   *
+   * @param datanodeDetails DatanodeDetails
+   * @param replicas List of ContainerReplicaProto
+   * @param publisher EventPublisher reference
+   */
+  private void updateDeleteTransaction(final DatanodeDetails datanodeDetails,
 
 Review comment:
   This is for updating the block deletion transaction Id reported by the container replica.
If the container replica is lagging behind in block deletion, we trigger PENDING_DELETE_STATUS
event so that SCMBlockDeletingService can resend block deletion commands to the datanode.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org


Mime
View raw message