hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [hadoop] branch ozone-0.4 updated: HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534)
Date Tue, 05 Mar 2019 19:49:20 GMT
This is an automated email from the ASF dual-hosted git repository.

ajay pushed a commit to branch ozone-0.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/ozone-0.4 by this push:
     new 4cf50e3  HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534)
4cf50e3 is described below

commit 4cf50e3972c3062c53a4bff50da085181c36cd72
Author: Bharat Viswanadham <bharat@apache.org>
AuthorDate: Tue Mar 5 11:46:36 2019 -0800

    HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534)
    
    * HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule.
    
    (cherry picked from commit 313e8b9f1344da376de2b0e899e78e384f37ff35)
---
 .../hdds/scm/chillmode/ContainerChillModeRule.java | 43 +++++++++++++++-------
 .../hdds/scm/chillmode/DataNodeChillModeRule.java  | 41 +++++++++++++++------
 .../hdds/scm/chillmode/SCMChillModeManager.java    | 30 ++++++---------
 .../hdds/scm/server/StorageContainerManager.java   |  2 -
 .../scm/chillmode/TestSCMChillModeManager.java     | 14 ++-----
 5 files changed, 75 insertions(+), 55 deletions(-)

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
index 9578532..17dd496 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
@@ -29,12 +29,15 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
 
 /**
  * Class defining Chill mode exit criteria for Containers.
  */
 public class ContainerChillModeRule implements
-    ChillModeExitRule<NodeRegistrationContainerReport> {
+    ChillModeExitRule<NodeRegistrationContainerReport>,
+    EventHandler<NodeRegistrationContainerReport> {
 
   // Required cutoff % for containers with at least 1 reported replica.
   private double chillModeCutoff;
@@ -68,9 +71,6 @@ public class ContainerChillModeRule implements
 
   @Override
   public boolean validate() {
-    if (maxContainer == 0) {
-      return true;
-    }
     return getCurrentContainerThreshold() >= chillModeCutoff;
   }
 
@@ -84,10 +84,6 @@ public class ContainerChillModeRule implements
 
   @Override
   public void process(NodeRegistrationContainerReport reportsProto) {
-    if (maxContainer == 0) {
-      // No container to check.
-      return;
-    }
 
     reportsProto.getReport().getReportsList().forEach(c -> {
       if (containerMap.containsKey(c.getContainerID())) {
@@ -96,12 +92,33 @@ public class ContainerChillModeRule implements
         }
       }
     });
-    if(chillModeManager.getInChillMode()) {
-      SCMChillModeManager.getLogger().info(
-          "SCM in chill mode. {} % containers have at least one"
-              + " reported replica.",
-          (containerWithMinReplicas.get() / maxContainer) * 100);
+  }
+
+  @Override
+  public void onMessage(NodeRegistrationContainerReport
+      nodeRegistrationContainerReport, EventPublisher publisher) {
+
+    // TODO: when we have remove handlers, we can remove getInChillmode check
+
+    if (chillModeManager.getInChillMode()) {
+      if (validate()) {
+        return;
+      }
+
+      process(nodeRegistrationContainerReport);
+      if (chillModeManager.getInChillMode()) {
+        SCMChillModeManager.getLogger().info(
+            "SCM in chill mode. {} % containers have at least one"
+                + " reported replica.",
+            (containerWithMinReplicas.get() / maxContainer) * 100);
+      }
+
+      if (validate()) {
+        chillModeManager.validateChillModeExitRules(publisher);
+      }
+
     }
+
   }
 
   @Override
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
index 3950505..be99962 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
@@ -25,13 +25,16 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
 
 /**
  * Class defining Chill mode exit criteria according to number of DataNodes
  * registered with SCM.
  */
 public class DataNodeChillModeRule implements
-    ChillModeExitRule<NodeRegistrationContainerReport> {
+    ChillModeExitRule<NodeRegistrationContainerReport>,
+    EventHandler<NodeRegistrationContainerReport> {
 
   // Min DataNodes required to exit chill mode.
   private int requiredDns;
@@ -62,17 +65,33 @@ public class DataNodeChillModeRule implements
 
   @Override
   public void process(NodeRegistrationContainerReport reportsProto) {
-    if (requiredDns == 0) {
-      // No dn check required.
-      return;
-    }
 
-    if(chillModeManager.getInChillMode()) {
-      registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
-      registeredDns = registeredDnSet.size();
-      SCMChillModeManager.getLogger().info(
-          "SCM in chill mode. {} DataNodes registered, {} required.",
-          registeredDns, requiredDns);
+    registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
+    registeredDns = registeredDnSet.size();
+
+  }
+
+  @Override
+  public void onMessage(NodeRegistrationContainerReport
+      nodeRegistrationContainerReport, EventPublisher publisher) {
+    // TODO: when we have remove handlers, we can remove getInChillmode check
+
+    if (chillModeManager.getInChillMode()) {
+      if (validate()) {
+        return;
+      }
+
+      process(nodeRegistrationContainerReport);
+
+      if (chillModeManager.getInChillMode()) {
+        SCMChillModeManager.getLogger().info(
+            "SCM in chill mode. {} DataNodes registered, {} required.",
+            registeredDns, requiredDns);
+      }
+
+      if (validate()) {
+        chillModeManager.validateChillModeExitRules(publisher);
+      }
     }
   }
 
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
index 1d97a57..8ced6d5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -28,9 +28,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
 import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
 import org.apache.hadoop.hdds.scm.pipeline.RatisPipelineUtils;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
-    .NodeRegistrationContainerReport;
-import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.slf4j.Logger;
@@ -48,8 +45,7 @@ import org.slf4j.LoggerFactory;
  * for reported containers and validates if cutoff threshold for
  * containers is meet.
  */
-public class SCMChillModeManager implements
-    EventHandler<NodeRegistrationContainerReport> {
+public class SCMChillModeManager {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(SCMChillModeManager.class);
@@ -78,9 +74,16 @@ public class SCMChillModeManager implements
         HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
         HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
     if (isChillModeEnabled) {
-      exitRules.put(CONT_EXIT_RULE,
-          new ContainerChillModeRule(config, allContainers, this));
-      exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+      ContainerChillModeRule containerChillModeRule =
+          new ContainerChillModeRule(config, allContainers, this);
+      DataNodeChillModeRule dataNodeChillModeRule =
+          new DataNodeChillModeRule(config, this);
+      exitRules.put(CONT_EXIT_RULE, containerChillModeRule);
+      exitRules.put(DN_EXIT_RULE, dataNodeChillModeRule);
+      eventPublisher.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+          containerChillModeRule);
+      eventPublisher.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+          dataNodeChillModeRule);
 
       if (conf.getBoolean(
           HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
@@ -146,17 +149,6 @@ public class SCMChillModeManager implements
         .scheduleFixedIntervalPipelineCreator(pipelineManager, config);
   }
 
-  @Override
-  public void onMessage(
-      NodeRegistrationContainerReport nodeRegistrationContainerReport,
-      EventPublisher publisher) {
-    if (getInChillMode()) {
-      exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport);
-      exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport);
-      validateChillModeExitRules(publisher);
-    }
-  }
-
   public boolean getInChillMode() {
     if (!isChillModeEnabled) {
       return false;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index fc93235..c9446a5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -364,8 +364,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
         replicationStatus.getChillModeStatusListener());
     eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
         (BlockManagerImpl) scmBlockManager);
-    eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
     registerMXBean();
   }
 
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
index faf8fee..957fe70 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -90,8 +90,7 @@ public class TestSCMChillModeManager {
     }
     scmChillModeManager = new SCMChillModeManager(
         config, containers, null, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
+
     assertTrue(scmChillModeManager.getInChillMode());
     queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
         HddsTestUtils.createNodeRegistrationContainerReport(containers));
@@ -111,8 +110,7 @@ public class TestSCMChillModeManager {
     }
     scmChillModeManager = new SCMChillModeManager(
         config, containers, null, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
+
     assertTrue(scmChillModeManager.getInChillMode());
 
     testContainerThreshold(containers.subList(0, 25), 0.25);
@@ -167,8 +165,7 @@ public class TestSCMChillModeManager {
 
     scmChillModeManager = new SCMChillModeManager(
         config, containers, null, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
+
     assertTrue(scmChillModeManager.getInChillMode());
 
     // When 10 CLOSED containers are reported by DNs, the computed container
@@ -192,8 +189,7 @@ public class TestSCMChillModeManager {
     conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
     scmChillModeManager = new SCMChillModeManager(
         conf, containers, null, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
+
     // Assert SCM is in Chill mode.
     assertTrue(scmChillModeManager.getInChillMode());
 
@@ -256,8 +252,6 @@ public class TestSCMChillModeManager {
 
       scmChillModeManager = new SCMChillModeManager(
           config, containers, pipelineManager, queue);
-      queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-          scmChillModeManager);
 
       queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
           HddsTestUtils.createNodeRegistrationContainerReport(containers));


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message