From common-commits-return-93557-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Mar 5 19:49:23 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 5CA97180648 for ; Tue, 5 Mar 2019 20:49:22 +0100 (CET) Received: (qmail 30945 invoked by uid 500); 5 Mar 2019 19:49:21 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 30930 invoked by uid 99); 5 Mar 2019 19:49:21 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Mar 2019 19:49:21 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id A678F877DA; Tue, 5 Mar 2019 19:49:20 +0000 (UTC) Date: Tue, 05 Mar 2019 19:49:20 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch ozone-0.4 updated: HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155181536052.4266.3956976386610264327@gitbox.apache.org> From: ajay@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/ozone-0.4 X-Git-Reftype: branch X-Git-Oldrev: c57e1ddd63973f7dce5a9a3170b0fc1a521e14ab X-Git-Newrev: 4cf50e3972c3062c53a4bff50da085181c36cd72 X-Git-Rev: 4cf50e3972c3062c53a4bff50da085181c36cd72 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. ajay pushed a commit to branch ozone-0.4 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/ozone-0.4 by this push: new 4cf50e3 HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534) 4cf50e3 is described below commit 4cf50e3972c3062c53a4bff50da085181c36cd72 Author: Bharat Viswanadham AuthorDate: Tue Mar 5 11:46:36 2019 -0800 HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (#534) * HDDS-1193. Refactor ContainerChillModeRule and DatanodeChillMode rule. (cherry picked from commit 313e8b9f1344da376de2b0e899e78e384f37ff35) --- .../hdds/scm/chillmode/ContainerChillModeRule.java | 43 +++++++++++++++------- .../hdds/scm/chillmode/DataNodeChillModeRule.java | 41 +++++++++++++++------ .../hdds/scm/chillmode/SCMChillModeManager.java | 30 ++++++--------- .../hdds/scm/server/StorageContainerManager.java | 2 - .../scm/chillmode/TestSCMChillModeManager.java | 14 ++----- 5 files changed, 75 insertions(+), 55 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java index 9578532..17dd496 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java @@ -29,12 +29,15 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; /** * Class defining Chill mode exit criteria for Containers. */ public class ContainerChillModeRule implements - ChillModeExitRule { + ChillModeExitRule, + EventHandler { // Required cutoff % for containers with at least 1 reported replica. private double chillModeCutoff; @@ -68,9 +71,6 @@ public class ContainerChillModeRule implements @Override public boolean validate() { - if (maxContainer == 0) { - return true; - } return getCurrentContainerThreshold() >= chillModeCutoff; } @@ -84,10 +84,6 @@ public class ContainerChillModeRule implements @Override public void process(NodeRegistrationContainerReport reportsProto) { - if (maxContainer == 0) { - // No container to check. - return; - } reportsProto.getReport().getReportsList().forEach(c -> { if (containerMap.containsKey(c.getContainerID())) { @@ -96,12 +92,33 @@ public class ContainerChillModeRule implements } } }); - if(chillModeManager.getInChillMode()) { - SCMChillModeManager.getLogger().info( - "SCM in chill mode. {} % containers have at least one" - + " reported replica.", - (containerWithMinReplicas.get() / maxContainer) * 100); + } + + @Override + public void onMessage(NodeRegistrationContainerReport + nodeRegistrationContainerReport, EventPublisher publisher) { + + // TODO: when we have remove handlers, we can remove getInChillmode check + + if (chillModeManager.getInChillMode()) { + if (validate()) { + return; + } + + process(nodeRegistrationContainerReport); + if (chillModeManager.getInChillMode()) { + SCMChillModeManager.getLogger().info( + "SCM in chill mode. {} % containers have at least one" + + " reported replica.", + (containerWithMinReplicas.get() / maxContainer) * 100); + } + + if (validate()) { + chillModeManager.validateChillModeExitRules(publisher); + } + } + } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java index 3950505..be99962 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java @@ -25,13 +25,16 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; /** * Class defining Chill mode exit criteria according to number of DataNodes * registered with SCM. */ public class DataNodeChillModeRule implements - ChillModeExitRule { + ChillModeExitRule, + EventHandler { // Min DataNodes required to exit chill mode. private int requiredDns; @@ -62,17 +65,33 @@ public class DataNodeChillModeRule implements @Override public void process(NodeRegistrationContainerReport reportsProto) { - if (requiredDns == 0) { - // No dn check required. - return; - } - if(chillModeManager.getInChillMode()) { - registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid()); - registeredDns = registeredDnSet.size(); - SCMChillModeManager.getLogger().info( - "SCM in chill mode. {} DataNodes registered, {} required.", - registeredDns, requiredDns); + registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid()); + registeredDns = registeredDnSet.size(); + + } + + @Override + public void onMessage(NodeRegistrationContainerReport + nodeRegistrationContainerReport, EventPublisher publisher) { + // TODO: when we have remove handlers, we can remove getInChillmode check + + if (chillModeManager.getInChillMode()) { + if (validate()) { + return; + } + + process(nodeRegistrationContainerReport); + + if (chillModeManager.getInChillMode()) { + SCMChillModeManager.getLogger().info( + "SCM in chill mode. {} DataNodes registered, {} required.", + registeredDns, requiredDns); + } + + if (validate()) { + chillModeManager.validateChillModeExitRules(publisher); + } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java index 1d97a57..8ced6d5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java @@ -28,9 +28,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.RatisPipelineUtils; -import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer - .NodeRegistrationContainerReport; -import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; import org.slf4j.Logger; @@ -48,8 +45,7 @@ import org.slf4j.LoggerFactory; * for reported containers and validates if cutoff threshold for * containers is meet. */ -public class SCMChillModeManager implements - EventHandler { +public class SCMChillModeManager { private static final Logger LOG = LoggerFactory.getLogger(SCMChillModeManager.class); @@ -78,9 +74,16 @@ public class SCMChillModeManager implements HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT); if (isChillModeEnabled) { - exitRules.put(CONT_EXIT_RULE, - new ContainerChillModeRule(config, allContainers, this)); - exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this)); + ContainerChillModeRule containerChillModeRule = + new ContainerChillModeRule(config, allContainers, this); + DataNodeChillModeRule dataNodeChillModeRule = + new DataNodeChillModeRule(config, this); + exitRules.put(CONT_EXIT_RULE, containerChillModeRule); + exitRules.put(DN_EXIT_RULE, dataNodeChillModeRule); + eventPublisher.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, + containerChillModeRule); + eventPublisher.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, + dataNodeChillModeRule); if (conf.getBoolean( HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK, @@ -146,17 +149,6 @@ public class SCMChillModeManager implements .scheduleFixedIntervalPipelineCreator(pipelineManager, config); } - @Override - public void onMessage( - NodeRegistrationContainerReport nodeRegistrationContainerReport, - EventPublisher publisher) { - if (getInChillMode()) { - exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport); - exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport); - validateChillModeExitRules(publisher); - } - } - public boolean getInChillMode() { if (!isChillModeEnabled) { return false; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index fc93235..c9446a5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -364,8 +364,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl replicationStatus.getChillModeStatusListener()); eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, (BlockManagerImpl) scmBlockManager); - eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, - scmChillModeManager); registerMXBean(); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java index faf8fee..957fe70 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java @@ -90,8 +90,7 @@ public class TestSCMChillModeManager { } scmChillModeManager = new SCMChillModeManager( config, containers, null, queue); - queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, - scmChillModeManager); + assertTrue(scmChillModeManager.getInChillMode()); queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, HddsTestUtils.createNodeRegistrationContainerReport(containers)); @@ -111,8 +110,7 @@ public class TestSCMChillModeManager { } scmChillModeManager = new SCMChillModeManager( config, containers, null, queue); - queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, - scmChillModeManager); + assertTrue(scmChillModeManager.getInChillMode()); testContainerThreshold(containers.subList(0, 25), 0.25); @@ -167,8 +165,7 @@ public class TestSCMChillModeManager { scmChillModeManager = new SCMChillModeManager( config, containers, null, queue); - queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, - scmChillModeManager); + assertTrue(scmChillModeManager.getInChillMode()); // When 10 CLOSED containers are reported by DNs, the computed container @@ -192,8 +189,7 @@ public class TestSCMChillModeManager { conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns); scmChillModeManager = new SCMChillModeManager( conf, containers, null, queue); - queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, - scmChillModeManager); + // Assert SCM is in Chill mode. assertTrue(scmChillModeManager.getInChillMode()); @@ -256,8 +252,6 @@ public class TestSCMChillModeManager { scmChillModeManager = new SCMChillModeManager( config, containers, pipelineManager, queue); - queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT, - scmChillModeManager); queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, HddsTestUtils.createNodeRegistrationContainerReport(containers)); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org