hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [03/49] hadoop git commit: HDDS-809. Refactor SCMChillModeManager.
Date Tue, 13 Nov 2018 07:50:30 GMT
HDDS-809. Refactor SCMChillModeManager.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/addec292
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/addec292
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/addec292

Branch: refs/heads/HDFS-13891
Commit: addec29297e61a417f0ce711bd76b6db53d504eb
Parents: 482716e
Author: Yiqun Lin <yqlin@apache.org>
Authored: Wed Nov 7 13:53:28 2018 +0800
Committer: Yiqun Lin <yqlin@apache.org>
Committed: Wed Nov 7 13:54:08 2018 +0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdds/scm/ScmUtils.java    |   2 +-
 .../hadoop/hdds/scm/block/BlockManagerImpl.java |   2 +-
 .../hdds/scm/chillmode/ChillModeExitRule.java   |  32 ++
 .../hdds/scm/chillmode/ChillModePrecheck.java   |  68 ++++
 .../scm/chillmode/ChillModeRestrictedOps.java   |  41 +++
 .../scm/chillmode/ContainerChillModeRule.java   | 112 +++++++
 .../scm/chillmode/DataNodeChillModeRule.java    |  83 +++++
 .../hadoop/hdds/scm/chillmode/Precheck.java     |  29 ++
 .../hdds/scm/chillmode/SCMChillModeManager.java | 153 +++++++++
 .../hadoop/hdds/scm/chillmode/package-info.java |  18 ++
 .../hdds/scm/server/ChillModePrecheck.java      |  69 ----
 .../apache/hadoop/hdds/scm/server/Precheck.java |  29 --
 .../hdds/scm/server/SCMChillModeManager.java    | 319 -------------------
 .../scm/server/SCMClientProtocolServer.java     |   1 +
 .../scm/server/StorageContainerManager.java     |   1 +
 .../scm/chillmode/TestSCMChillModeManager.java  | 215 +++++++++++++
 .../scm/server/TestSCMChillModeManager.java     | 215 -------------
 .../hadoop/ozone/om/TestScmChillMode.java       |   2 +-
 18 files changed, 756 insertions(+), 635 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
index 435f0a5..43b4452 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -19,8 +19,8 @@
 package org.apache.hadoop.hdds.scm;
 
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+import org.apache.hadoop.hdds.scm.chillmode.Precheck;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.server.Precheck;
 
 /**
  * SCM utility class.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index c878d97..85658b9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.client.ContainerBlockID;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.chillmode.ChillModePrecheck;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -32,7 +33,6 @@ import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.server.ChillModePrecheck;
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.metrics2.util.MBeans;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java
new file mode 100644
index 0000000..d283dfe
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+/**
+ * Interface for defining chill mode exit rules.
+ *
+ * @param <T>
+ */
+public interface ChillModeExitRule<T> {
+
+  boolean validate();
+
+  void process(T report);
+
+  void cleanup();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java
new file mode 100644
index 0000000..0ed06dd
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
+
+/**
+ * Chill mode pre-check for SCM operations.
+ * */
+public class ChillModePrecheck implements Precheck<ScmOps> {
+
+  private AtomicBoolean inChillMode;
+  public static final String PRECHECK_TYPE = "ChillModePrecheck";
+
+  public ChillModePrecheck(Configuration conf) {
+    boolean chillModeEnabled = conf.getBoolean(
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
+    if (chillModeEnabled) {
+      inChillMode = new AtomicBoolean(true);
+    } else {
+      inChillMode = new AtomicBoolean(false);
+    }
+  }
+
+  public boolean check(ScmOps op) throws SCMException {
+    if (inChillMode.get() && ChillModeRestrictedOps
+        .isRestrictedInChillMode(op)) {
+      throw new SCMException("ChillModePrecheck failed for " + op,
+          ResultCodes.CHILL_MODE_EXCEPTION);
+    }
+    return inChillMode.get();
+  }
+
+  @Override
+  public String type() {
+    return PRECHECK_TYPE;
+  }
+
+  public boolean isInChillMode() {
+    return inChillMode.get();
+  }
+
+  public void setInChillMode(boolean inChillMode) {
+    this.inChillMode.set(inChillMode);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java
new file mode 100644
index 0000000..f3b003b
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.EnumSet;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+
+/**
+ * Operations restricted in SCM chill mode.
+ */
+public final class ChillModeRestrictedOps {
+  private static EnumSet restrictedOps = EnumSet.noneOf(ScmOps.class);
+
+  private ChillModeRestrictedOps() {
+  }
+
+  static {
+    restrictedOps.add(ScmOps.allocateBlock);
+    restrictedOps.add(ScmOps.allocateContainer);
+  }
+
+  public static boolean isRestrictedInChillMode(ScmOps opName) {
+    return restrictedOps.contains(opName);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
new file mode 100644
index 0000000..57eb8dd
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Class defining Chill mode exit criteria for Containers.
+ */
+public class ContainerChillModeRule implements
+    ChillModeExitRule<NodeRegistrationContainerReport> {
+
+  // Required cutoff % for containers with at least 1 reported replica.
+  private double chillModeCutoff;
+  // Containers read from scm db (excluding containers in ALLOCATED state).
+  private Map<Long, ContainerInfo> containerMap;
+  private double maxContainer;
+
+  private AtomicLong containerWithMinReplicas = new AtomicLong(0);
+  private final SCMChillModeManager chillModeManager;
+
+  public ContainerChillModeRule(Configuration conf,
+      List<ContainerInfo> containers, SCMChillModeManager manager) {
+    chillModeCutoff = conf.getDouble(
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT,
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT);
+    chillModeManager = manager;
+    containerMap = new ConcurrentHashMap<>();
+    if(containers != null) {
+      containers.forEach(c -> {
+        // Containers in ALLOCATED state should not be included while
+        // calculating the total number of containers here. They are not
+        // reported by DNs and hence should not affect the chill mode exit
+        // rule.
+        if (c != null && c.getState() != null &&
+            !c.getState().equals(HddsProtos.LifeCycleState.ALLOCATED)) {
+          containerMap.put(c.getContainerID(), c);
+        }
+      });
+      maxContainer = containerMap.size();
+    }
+  }
+
+  @Override
+  public boolean validate() {
+    if (maxContainer == 0) {
+      return true;
+    }
+    return getCurrentContainerThreshold() >= chillModeCutoff;
+  }
+
+  @VisibleForTesting
+  public double getCurrentContainerThreshold() {
+    if (maxContainer == 0) {
+      return 1;
+    }
+    return (containerWithMinReplicas.doubleValue() / maxContainer);
+  }
+
+  @Override
+  public void process(NodeRegistrationContainerReport reportsProto) {
+    if (maxContainer == 0) {
+      // No container to check.
+      return;
+    }
+
+    reportsProto.getReport().getReportsList().forEach(c -> {
+      if (containerMap.containsKey(c.getContainerID())) {
+        if(containerMap.remove(c.getContainerID()) != null) {
+          containerWithMinReplicas.getAndAdd(1);
+        }
+      }
+    });
+    if(chillModeManager.getInChillMode()) {
+      SCMChillModeManager.getLogger().info(
+          "SCM in chill mode. {} % containers have at least one"
+              + " reported replica.",
+          (containerWithMinReplicas.get() / maxContainer) * 100);
+    }
+  }
+
+  @Override
+  public void cleanup() {
+    containerMap.clear();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
new file mode 100644
index 0000000..3950505
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.HashSet;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Class defining Chill mode exit criteria according to number of DataNodes
+ * registered with SCM.
+ */
+public class DataNodeChillModeRule implements
+    ChillModeExitRule<NodeRegistrationContainerReport> {
+
+  // Min DataNodes required to exit chill mode.
+  private int requiredDns;
+  private int registeredDns = 0;
+  // Set to track registered DataNodes.
+  private HashSet<UUID> registeredDnSet;
+
+  private final SCMChillModeManager chillModeManager;
+
+  public DataNodeChillModeRule(Configuration conf,
+      SCMChillModeManager manager) {
+    requiredDns = conf.getInt(
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE,
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT);
+    registeredDnSet = new HashSet<>(requiredDns * 2);
+    chillModeManager = manager;
+  }
+
+  @Override
+  public boolean validate() {
+    return registeredDns >= requiredDns;
+  }
+
+  @VisibleForTesting
+  public double getRegisteredDataNodes() {
+    return registeredDns;
+  }
+
+  @Override
+  public void process(NodeRegistrationContainerReport reportsProto) {
+    if (requiredDns == 0) {
+      // No dn check required.
+      return;
+    }
+
+    if(chillModeManager.getInChillMode()) {
+      registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
+      registeredDns = registeredDnSet.size();
+      SCMChillModeManager.getLogger().info(
+          "SCM in chill mode. {} DataNodes registered, {} required.",
+          registeredDns, requiredDns);
+    }
+  }
+
+  @Override
+  public void cleanup() {
+    registeredDnSet.clear();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java
new file mode 100644
index 0000000..30daa85
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+
+/**
+ * Precheck for SCM operations.
+ * */
+public interface Precheck<T> {
+  boolean check(T t) throws SCMException;
+  String type();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
new file mode 100644
index 0000000..fa0eaa9
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
+    .NodeRegistrationContainerReport;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * StorageContainerManager enters chill mode on startup to allow system to
+ * reach a stable state before becoming fully functional. SCM will wait
+ * for certain resources to be reported before coming out of chill mode.
+ *
+ * ChillModeExitRule defines format to define new rules which must be satisfied
+ * to exit Chill mode.
+ * ContainerChillModeRule defines the only exit criteria right now.
+ * On every new datanode registration event this class adds replicas
+ * for reported containers and validates if cutoff threshold for
+ * containers is meet.
+ */
+public class SCMChillModeManager implements
+    EventHandler<NodeRegistrationContainerReport> {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SCMChillModeManager.class);
+  private final boolean isChillModeEnabled;
+  private AtomicBoolean inChillMode = new AtomicBoolean(true);
+  private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
+  private Configuration config;
+  private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
+  private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
+  private final EventQueue eventPublisher;
+
+  public SCMChillModeManager(Configuration conf,
+      List<ContainerInfo> allContainers, EventQueue eventQueue) {
+    this.config = conf;
+    this.eventPublisher = eventQueue;
+    this.isChillModeEnabled = conf.getBoolean(
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
+        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
+    if (isChillModeEnabled) {
+      exitRules.put(CONT_EXIT_RULE,
+          new ContainerChillModeRule(config, allContainers, this));
+      exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+      emitChillModeStatus();
+    } else {
+      exitChillMode(eventQueue);
+    }
+  }
+
+  /**
+   * Emit Chill mode status.
+   */
+  @VisibleForTesting
+  public void emitChillModeStatus() {
+    eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
+  }
+
+  private void validateChillModeExitRules(EventPublisher eventQueue) {
+    for (ChillModeExitRule exitRule : exitRules.values()) {
+      if (!exitRule.validate()) {
+        return;
+      }
+    }
+    exitChillMode(eventQueue);
+  }
+
+  /**
+   * Exit chill mode. It does following actions:
+   * 1. Set chill mode status to false.
+   * 2. Emits START_REPLICATION for ReplicationManager.
+   * 3. Cleanup resources.
+   * 4. Emit chill mode status.
+   * @param eventQueue
+   */
+  @VisibleForTesting
+  public void exitChillMode(EventPublisher eventQueue) {
+    LOG.info("SCM exiting chill mode.");
+    setInChillMode(false);
+
+    // TODO: Remove handler registration as there is no need to listen to
+    // register events anymore.
+
+    for (ChillModeExitRule e : exitRules.values()) {
+      e.cleanup();
+    }
+    emitChillModeStatus();
+  }
+
+  @Override
+  public void onMessage(
+      NodeRegistrationContainerReport nodeRegistrationContainerReport,
+      EventPublisher publisher) {
+    if (getInChillMode()) {
+      exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport);
+      exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport);
+      validateChillModeExitRules(publisher);
+    }
+  }
+
+  public boolean getInChillMode() {
+    if (!isChillModeEnabled) {
+      return false;
+    }
+    return inChillMode.get();
+  }
+
+  /**
+   * Set chill mode status.
+   */
+  public void setInChillMode(boolean inChillMode) {
+    this.inChillMode.set(inChillMode);
+  }
+
+  public static Logger getLogger() {
+    return LOG;
+  }
+
+  @VisibleForTesting
+  public double getCurrentContainerThreshold() {
+    return ((ContainerChillModeRule) exitRules.get(CONT_EXIT_RULE))
+        .getCurrentContainerThreshold();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java
new file mode 100644
index 0000000..2513ac7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
deleted file mode 100644
index a3f4c90..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.server;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
-import org.apache.hadoop.hdds.scm.server.SCMChillModeManager.ChillModeRestrictedOps;
-
-/**
- * Chill mode pre-check for SCM operations.
- * */
-public class ChillModePrecheck implements Precheck<ScmOps> {
-
-  private AtomicBoolean inChillMode;
-  public static final String PRECHECK_TYPE = "ChillModePrecheck";
-
-  public ChillModePrecheck(Configuration conf) {
-    boolean chillModeEnabled = conf.getBoolean(
-        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
-        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
-    if (chillModeEnabled) {
-      inChillMode = new AtomicBoolean(true);
-    } else {
-      inChillMode = new AtomicBoolean(false);
-    }
-  }
-
-  public boolean check(ScmOps op) throws SCMException {
-    if (inChillMode.get() && ChillModeRestrictedOps
-        .isRestrictedInChillMode(op)) {
-      throw new SCMException("ChillModePrecheck failed for " + op,
-          ResultCodes.CHILL_MODE_EXCEPTION);
-    }
-    return inChillMode.get();
-  }
-
-  @Override
-  public String type() {
-    return PRECHECK_TYPE;
-  }
-
-  public boolean isInChillMode() {
-    return inChillMode.get();
-  }
-
-  public void setInChillMode(boolean inChillMode) {
-    this.inChillMode.set(inChillMode);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java
deleted file mode 100644
index 1654990..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.server;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-
-/**
- * Precheck for SCM operations.
- * */
-public interface Precheck<T> {
-  boolean check(T t) throws SCMException;
-  String type();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
deleted file mode 100644
index 6a342d4..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.server;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
-    .NodeRegistrationContainerReport;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * StorageContainerManager enters chill mode on startup to allow system to
- * reach a stable state before becoming fully functional. SCM will wait
- * for certain resources to be reported before coming out of chill mode.
- *
- * ChillModeExitRule defines format to define new rules which must be satisfied
- * to exit Chill mode.
- * ContainerChillModeRule defines the only exit criteria right now.
- * On every new datanode registration event this class adds replicas
- * for reported containers and validates if cutoff threshold for
- * containers is meet.
- */
-public class SCMChillModeManager implements
-    EventHandler<NodeRegistrationContainerReport> {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SCMChillModeManager.class);
-  private final boolean isChillModeEnabled;
-  private AtomicBoolean inChillMode = new AtomicBoolean(true);
-  private AtomicLong containerWithMinReplicas = new AtomicLong(0);
-  private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
-  private Configuration config;
-  private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
-  private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
-  private final EventQueue eventPublisher;
-
-  SCMChillModeManager(Configuration conf, List<ContainerInfo> allContainers,
-      EventQueue eventQueue) {
-    this.config = conf;
-    this.eventPublisher = eventQueue;
-    this.isChillModeEnabled = conf.getBoolean(
-        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
-        HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
-    if (isChillModeEnabled) {
-      exitRules.put(CONT_EXIT_RULE,
-          new ContainerChillModeRule(config, allContainers));
-      exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config));
-      emitChillModeStatus();
-    } else {
-      exitChillMode(eventQueue);
-    }
-  }
-
-  /**
-   * Emit Chill mode status.
-   */
-  @VisibleForTesting
-  public void emitChillModeStatus() {
-    eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
-  }
-
-  private void validateChillModeExitRules(EventPublisher eventQueue) {
-    for (ChillModeExitRule exitRule : exitRules.values()) {
-      if (!exitRule.validate()) {
-        return;
-      }
-    }
-    exitChillMode(eventQueue);
-  }
-
-  /**
-   * Exit chill mode. It does following actions:
-   * 1. Set chill mode status to false.
-   * 2. Emits START_REPLICATION for ReplicationManager.
-   * 3. Cleanup resources.
-   * 4. Emit chill mode status.
-   * @param eventQueue
-   */
-  @VisibleForTesting
-  public void exitChillMode(EventPublisher eventQueue) {
-    LOG.info("SCM exiting chill mode.");
-    setInChillMode(false);
-
-    // TODO: Remove handler registration as there is no need to listen to
-    // register events anymore.
-
-    for (ChillModeExitRule e : exitRules.values()) {
-      e.cleanup();
-    }
-    emitChillModeStatus();
-  }
-
-  @Override
-  public void onMessage(
-      NodeRegistrationContainerReport nodeRegistrationContainerReport,
-      EventPublisher publisher) {
-    if (getInChillMode()) {
-      exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport);
-      exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport);
-      validateChillModeExitRules(publisher);
-    }
-  }
-
-  public boolean getInChillMode() {
-    if (!isChillModeEnabled) {
-      return false;
-    }
-    return inChillMode.get();
-  }
-
-  /**
-   * Set chill mode status.
-   */
-  public void setInChillMode(boolean inChillMode) {
-    this.inChillMode.set(inChillMode);
-  }
-
-  /**
-   * Interface for defining chill mode exit rules.
-   *
-   * @param <T>
-   */
-  public interface ChillModeExitRule<T> {
-
-    boolean validate();
-
-    void process(T report);
-
-    void cleanup();
-  }
-
-  /**
-   * Class defining Chill mode exit criteria for Containers.
-   */
-  public class ContainerChillModeRule implements
-      ChillModeExitRule<NodeRegistrationContainerReport> {
-
-    // Required cutoff % for containers with at least 1 reported replica.
-    private double chillModeCutoff;
-    // Containers read from scm db (excluding containers in ALLOCATED state).
-    private Map<Long, ContainerInfo> containerMap;
-    private double maxContainer;
-
-    public ContainerChillModeRule(Configuration conf,
-        List<ContainerInfo> containers) {
-      chillModeCutoff = conf
-          .getDouble(HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT,
-              HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT);
-      containerMap = new ConcurrentHashMap<>();
-      if(containers != null) {
-        containers.forEach(c -> {
-          // Containers in ALLOCATED state should not be included while
-          // calculating the total number of containers here. They are not
-          // reported by DNs and hence should not affect the chill mode exit
-          // rule.
-          if (c != null && c.getState() != null &&
-              !c.getState().equals(HddsProtos.LifeCycleState.ALLOCATED)) {
-            containerMap.put(c.getContainerID(), c);
-          }
-        });
-        maxContainer = containerMap.size();
-      }
-    }
-
-    @Override
-    public boolean validate() {
-      if (maxContainer == 0) {
-        return true;
-      }
-      return getCurrentContainerThreshold() >= chillModeCutoff;
-    }
-
-    @VisibleForTesting
-    public double getCurrentContainerThreshold() {
-      if (maxContainer == 0) {
-        return 1;
-      }
-      return (containerWithMinReplicas.doubleValue() / maxContainer);
-    }
-
-    @Override
-    public void process(NodeRegistrationContainerReport reportsProto) {
-      if (maxContainer == 0) {
-        // No container to check.
-        return;
-      }
-
-      reportsProto.getReport().getReportsList().forEach(c -> {
-        if (containerMap.containsKey(c.getContainerID())) {
-          if(containerMap.remove(c.getContainerID()) != null) {
-            containerWithMinReplicas.getAndAdd(1);
-          }
-        }
-      });
-      if(getInChillMode()) {
-        LOG.info("SCM in chill mode. {} % containers have at least one"
-                + " reported replica.",
-            (containerWithMinReplicas.get() / maxContainer) * 100);
-      }
-    }
-
-    @Override
-    public void cleanup() {
-      containerMap.clear();
-    }
-  }
-
-  /**
-   * Class defining Chill mode exit criteria according to number of DataNodes
-   * registered with SCM.
-   */
-  public class DataNodeChillModeRule implements
-      ChillModeExitRule<NodeRegistrationContainerReport> {
-
-    // Min DataNodes required to exit chill mode.
-    private int requiredDns;
-    private int registeredDns = 0;
-    // Set to track registered DataNodes.
-    private HashSet<UUID> registeredDnSet;
-
-    public DataNodeChillModeRule(Configuration conf) {
-      requiredDns = conf
-          .getInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE,
-              HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT);
-      registeredDnSet = new HashSet<>(requiredDns * 2);
-    }
-
-    @Override
-    public boolean validate() {
-      return registeredDns >= requiredDns;
-    }
-
-    @VisibleForTesting
-    public double getRegisteredDataNodes() {
-      return registeredDns;
-    }
-
-    @Override
-    public void process(NodeRegistrationContainerReport reportsProto) {
-      if (requiredDns == 0) {
-        // No dn check required.
-        return;
-      }
-
-      if(getInChillMode()) {
-        registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
-        registeredDns = registeredDnSet.size();
-        LOG.info("SCM in chill mode. {} DataNodes registered, {} required.",
-            registeredDns, requiredDns);
-      }
-    }
-
-    @Override
-    public void cleanup() {
-      registeredDnSet.clear();
-    }
-  }
-
-  @VisibleForTesting
-  public static Logger getLogger() {
-    return LOG;
-  }
-
-  @VisibleForTesting
-  public double getCurrentContainerThreshold() {
-    return ((ContainerChillModeRule) exitRules.get(CONT_EXIT_RULE))
-        .getCurrentContainerThreshold();
-  }
-
-  /**
-   * Operations restricted in SCM chill mode.
-   */
-  public static class ChillModeRestrictedOps {
-    private static EnumSet restrictedOps =  EnumSet.noneOf(ScmOps.class);
-
-    static {
-      restrictedOps.add(ScmOps.allocateBlock);
-      restrictedOps.add(ScmOps.allocateContainer);
-    }
-
-    public static boolean isRestrictedInChillMode(ScmOps opName) {
-      return restrictedOps.contains(opName);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 58cb871..181bcc5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.protocol.proto
 import org.apache.hadoop.hdds.scm.HddsServerUtil;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.chillmode.ChillModePrecheck;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index d8b9958..c1409cb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.block.BlockManager;
 import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
 import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
 import org.apache.hadoop.hdds.scm.block.PendingDeleteHandler;
+import org.apache.hadoop.hdds.scm.chillmode.SCMChillModeManager;
 import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
 import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
 import org.apache.hadoop.hdds.scm.container.CloseContainerWatcher;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
new file mode 100644
index 0000000..2c08077
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+/** Test class for SCMChillModeManager.
+ */
+public class TestSCMChillModeManager {
+
+  private static EventQueue queue;
+  private SCMChillModeManager scmChillModeManager;
+  private static Configuration config;
+  private List<ContainerInfo> containers;
+
+  @Rule
+  public Timeout timeout = new Timeout(1000 * 35);
+
+  @BeforeClass
+  public static void setUp() {
+    queue = new EventQueue();
+    config = new OzoneConfiguration();
+  }
+
+  @Test
+  public void testChillModeState() throws Exception {
+    // Test 1: test for 0 containers
+    testChillMode(0);
+
+    // Test 2: test for 20 containers
+    testChillMode(20);
+  }
+
+  @Test
+  public void testChillModeStateWithNullContainers() {
+    new SCMChillModeManager(config, null, queue);
+  }
+
+  private void testChillMode(int numContainers) throws Exception {
+    containers = new ArrayList<>();
+    containers.addAll(HddsTestUtils.getContainerInfo(numContainers));
+    // Assign open state to containers to be included in the chill mode
+    // container list
+    for (ContainerInfo container : containers) {
+      container.setState(HddsProtos.LifeCycleState.OPEN);
+    }
+    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        scmChillModeManager);
+    assertTrue(scmChillModeManager.getInChillMode());
+    queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        HddsTestUtils.createNodeRegistrationContainerReport(containers));
+    GenericTestUtils.waitFor(() -> {
+      return !scmChillModeManager.getInChillMode();
+    }, 100, 1000 * 5);
+  }
+
+  @Test
+  public void testChillModeExitRule() throws Exception {
+    containers = new ArrayList<>();
+    containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+    // Assign open state to containers to be included in the chill mode
+    // container list
+    for (ContainerInfo container : containers) {
+      container.setState(HddsProtos.LifeCycleState.OPEN);
+    }
+    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        scmChillModeManager);
+    assertTrue(scmChillModeManager.getInChillMode());
+
+    testContainerThreshold(containers.subList(0, 25), 0.25);
+    assertTrue(scmChillModeManager.getInChillMode());
+    testContainerThreshold(containers.subList(25, 50), 0.50);
+    assertTrue(scmChillModeManager.getInChillMode());
+    testContainerThreshold(containers.subList(50, 75), 0.75);
+    assertTrue(scmChillModeManager.getInChillMode());
+    testContainerThreshold(containers.subList(75, 100), 1.0);
+
+    GenericTestUtils.waitFor(() -> {
+      return !scmChillModeManager.getInChillMode();
+    }, 100, 1000 * 5);
+  }
+
+  @Test
+  public void testDisableChillMode() {
+    OzoneConfiguration conf = new OzoneConfiguration(config);
+    conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
+    scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+    assertFalse(scmChillModeManager.getInChillMode());
+  }
+
+  @Test
+  public void testChillModeDataNodeExitRule() throws Exception {
+    containers = new ArrayList<>();
+    testChillModeDataNodes(0);
+    testChillModeDataNodes(3);
+    testChillModeDataNodes(5);
+  }
+
+  /**
+   * Check that containers in Allocated state are not considered while
+   * computing percentage of containers with at least 1 reported replica in
+   * chill mode exit rule.
+   */
+  @Test
+  public void testContainerChillModeRule() throws Exception {
+    containers = new ArrayList<>();
+    // Add 100 containers to the list of containers in SCM
+    containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+    // Assign OPEN state to first 25 containers and ALLLOCATED state to rest
+    // of the containers
+    for (ContainerInfo container : containers.subList(0, 25)) {
+      container.setState(HddsProtos.LifeCycleState.OPEN);
+    }
+    for (ContainerInfo container : containers.subList(25, 100)) {
+      container.setState(HddsProtos.LifeCycleState.ALLOCATED);
+    }
+
+    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        scmChillModeManager);
+    assertTrue(scmChillModeManager.getInChillMode());
+
+    // When 10 OPEN containers are reported by DNs, the computed container
+    // threshold should be 10/25 as there are only 25 open containers.
+    // Containers in ALLOCATED state should not contribute towards list of
+    // containers while calculating container threshold in SCMChillNodeManager
+    testContainerThreshold(containers.subList(0, 10), 0.4);
+    assertTrue(scmChillModeManager.getInChillMode());
+
+    // When remaining 15 OPEN containers are reported by DNs, the container
+    // threshold should be (10+15)/25.
+    testContainerThreshold(containers.subList(10, 25), 1.0);
+
+    GenericTestUtils.waitFor(() -> {
+      return !scmChillModeManager.getInChillMode();
+    }, 100, 1000 * 5);
+  }
+
+  private void testChillModeDataNodes(int numOfDns) throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration(config);
+    conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
+    scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        scmChillModeManager);
+    // Assert SCM is in Chill mode.
+    assertTrue(scmChillModeManager.getInChillMode());
+
+    // Register all DataNodes except last one and assert SCM is in chill mode.
+    for (int i = 0; i < numOfDns-1; i++) {
+      queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+          HddsTestUtils.createNodeRegistrationContainerReport(containers));
+      assertTrue(scmChillModeManager.getInChillMode());
+      assertTrue(scmChillModeManager.getCurrentContainerThreshold() == 1);
+    }
+
+    if(numOfDns == 0){
+      GenericTestUtils.waitFor(() -> {
+        return scmChillModeManager.getInChillMode();
+      }, 10, 1000 * 10);
+      return;
+    }
+    // Register last DataNode and check that SCM is out of Chill mode.
+    queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        HddsTestUtils.createNodeRegistrationContainerReport(containers));
+    GenericTestUtils.waitFor(() -> {
+      return !scmChillModeManager.getInChillMode();
+    }, 10, 1000 * 10);
+  }
+
+  private void testContainerThreshold(List<ContainerInfo> dnContainers,
+      double expectedThreshold)
+      throws Exception {
+    queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+        HddsTestUtils.createNodeRegistrationContainerReport(dnContainers));
+    GenericTestUtils.waitFor(() -> {
+      double threshold = scmChillModeManager.getCurrentContainerThreshold();
+      return threshold == expectedThreshold;
+    }, 100, 2000 * 9);
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
deleted file mode 100644
index ef28354..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.server;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.HddsTestUtils;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-/** Test class for SCMChillModeManager.
- */
-public class TestSCMChillModeManager {
-
-  private static EventQueue queue;
-  private SCMChillModeManager scmChillModeManager;
-  private static Configuration config;
-  private List<ContainerInfo> containers;
-
-  @Rule
-  public Timeout timeout = new Timeout(1000 * 35);
-
-  @BeforeClass
-  public static void setUp() {
-    queue = new EventQueue();
-    config = new OzoneConfiguration();
-  }
-
-  @Test
-  public void testChillModeState() throws Exception {
-    // Test 1: test for 0 containers
-    testChillMode(0);
-
-    // Test 2: test for 20 containers
-    testChillMode(20);
-  }
-
-  @Test
-  public void testChillModeStateWithNullContainers() {
-    new SCMChillModeManager(config, null, queue);
-  }
-
-  private void testChillMode(int numContainers) throws Exception {
-    containers = new ArrayList<>();
-    containers.addAll(HddsTestUtils.getContainerInfo(numContainers));
-    // Assign open state to containers to be included in the chill mode
-    // container list
-    for (ContainerInfo container : containers) {
-      container.setState(HddsProtos.LifeCycleState.OPEN);
-    }
-    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
-    assertTrue(scmChillModeManager.getInChillMode());
-    queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        HddsTestUtils.createNodeRegistrationContainerReport(containers));
-    GenericTestUtils.waitFor(() -> {
-      return !scmChillModeManager.getInChillMode();
-    }, 100, 1000 * 5);
-  }
-
-  @Test
-  public void testChillModeExitRule() throws Exception {
-    containers = new ArrayList<>();
-    containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
-    // Assign open state to containers to be included in the chill mode
-    // container list
-    for (ContainerInfo container : containers) {
-      container.setState(HddsProtos.LifeCycleState.OPEN);
-    }
-    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
-    assertTrue(scmChillModeManager.getInChillMode());
-
-    testContainerThreshold(containers.subList(0, 25), 0.25);
-    assertTrue(scmChillModeManager.getInChillMode());
-    testContainerThreshold(containers.subList(25, 50), 0.50);
-    assertTrue(scmChillModeManager.getInChillMode());
-    testContainerThreshold(containers.subList(50, 75), 0.75);
-    assertTrue(scmChillModeManager.getInChillMode());
-    testContainerThreshold(containers.subList(75, 100), 1.0);
-
-    GenericTestUtils.waitFor(() -> {
-      return !scmChillModeManager.getInChillMode();
-    }, 100, 1000 * 5);
-  }
-
-  @Test
-  public void testDisableChillMode() {
-    OzoneConfiguration conf = new OzoneConfiguration(config);
-    conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
-    scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
-    assertFalse(scmChillModeManager.getInChillMode());
-  }
-
-  @Test
-  public void testChillModeDataNodeExitRule() throws Exception {
-    containers = new ArrayList<>();
-    testChillModeDataNodes(0);
-    testChillModeDataNodes(3);
-    testChillModeDataNodes(5);
-  }
-
-  /**
-   * Check that containers in Allocated state are not considered while
-   * computing percentage of containers with at least 1 reported replica in
-   * chill mode exit rule.
-   */
-  @Test
-  public void testContainerChillModeRule() throws Exception {
-    containers = new ArrayList<>();
-    // Add 100 containers to the list of containers in SCM
-    containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
-    // Assign OPEN state to first 25 containers and ALLLOCATED state to rest
-    // of the containers
-    for (ContainerInfo container : containers.subList(0, 25)) {
-      container.setState(HddsProtos.LifeCycleState.OPEN);
-    }
-    for (ContainerInfo container : containers.subList(25, 100)) {
-      container.setState(HddsProtos.LifeCycleState.ALLOCATED);
-    }
-
-    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
-    assertTrue(scmChillModeManager.getInChillMode());
-
-    // When 10 OPEN containers are reported by DNs, the computed container
-    // threshold should be 10/25 as there are only 25 open containers.
-    // Containers in ALLOCATED state should not contribute towards list of
-    // containers while calculating container threshold in SCMChillNodeManager
-    testContainerThreshold(containers.subList(0, 10), 0.4);
-    assertTrue(scmChillModeManager.getInChillMode());
-
-    // When remaining 15 OPEN containers are reported by DNs, the container
-    // threshold should be (10+15)/25.
-    testContainerThreshold(containers.subList(10, 25), 1.0);
-
-    GenericTestUtils.waitFor(() -> {
-      return !scmChillModeManager.getInChillMode();
-    }, 100, 1000 * 5);
-  }
-
-  private void testChillModeDataNodes(int numOfDns) throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration(config);
-    conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
-    scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
-    queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        scmChillModeManager);
-    // Assert SCM is in Chill mode.
-    assertTrue(scmChillModeManager.getInChillMode());
-
-    // Register all DataNodes except last one and assert SCM is in chill mode.
-    for (int i = 0; i < numOfDns-1; i++) {
-      queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-          HddsTestUtils.createNodeRegistrationContainerReport(containers));
-      assertTrue(scmChillModeManager.getInChillMode());
-      assertTrue(scmChillModeManager.getCurrentContainerThreshold() == 1);
-    }
-
-    if(numOfDns == 0){
-      GenericTestUtils.waitFor(() -> {
-        return scmChillModeManager.getInChillMode();
-      }, 10, 1000 * 10);
-      return;
-    }
-    // Register last DataNode and check that SCM is out of Chill mode.
-    queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        HddsTestUtils.createNodeRegistrationContainerReport(containers));
-    GenericTestUtils.waitFor(() -> {
-      return !scmChillModeManager.getInChillMode();
-    }, 10, 1000 * 10);
-  }
-
-  private void testContainerThreshold(List<ContainerInfo> dnContainers,
-      double expectedThreshold)
-      throws Exception {
-    queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
-        HddsTestUtils.createNodeRegistrationContainerReport(dnContainers));
-    GenericTestUtils.waitFor(() -> {
-      double threshold = scmChillModeManager.getCurrentContainerThreshold();
-      return threshold == expectedThreshold;
-    }, 100, 2000 * 9);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
index dfcda5f..597bcf6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
@@ -30,11 +30,11 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
+import org.apache.hadoop.hdds.scm.chillmode.SCMChillModeManager;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.scm.server.SCMChillModeManager;
 import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.ozone.HddsDatanodeService;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message