hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [16/50] hadoop git commit: HDDS-259. Implement ContainerReportPublisher and NodeReportPublisher. Contributed by Nanda kumar.
Date Wed, 25 Jul 2018 01:31:59 GMT
HDDS-259. Implement ContainerReportPublisher and NodeReportPublisher. Contributed by Nanda
kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68b57ad3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68b57ad3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68b57ad3

Branch: refs/heads/HADOOP-15461
Commit: 68b57ad32cb0978ad5cd20b5fdc821f087a2c9dc
Parents: e9c44ec
Author: Xiaoyu Yao <xyao@apache.org>
Authored: Fri Jul 20 09:07:58 2018 -0700
Committer: Xiaoyu Yao <xyao@apache.org>
Committed: Fri Jul 20 09:12:48 2018 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  | 26 ++++++++++++++--
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  5 ---
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |  5 ---
 .../common/src/main/resources/ozone-default.xml | 26 +++++++++++++---
 .../apache/hadoop/hdds/scm/HddsServerUtil.java  | 13 ++++----
 .../report/CommandStatusReportPublisher.java    | 24 ++++++++++++---
 .../common/report/ContainerReportPublisher.java | 25 ++++++++++++---
 .../common/report/NodeReportPublisher.java      | 32 ++++++++++++++++++--
 .../common/report/ReportPublisher.java          | 14 +++++++--
 .../common/report/TestReportPublisher.java      | 11 ++-----
 .../scm/container/closer/ContainerCloser.java   | 12 ++++----
 .../container/closer/TestContainerCloser.java   |  8 ++---
 .../hadoop/hdds/scm/node/TestNodeManager.java   | 17 +++++------
 .../hadoop/ozone/MiniOzoneClusterImpl.java      |  5 +--
 .../ozone/TestStorageContainerManager.java      |  8 ++---
 .../hadoop/ozone/scm/node/TestQueryNode.java    |  5 ++-
 16 files changed, 162 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 8b449fb..0283615 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -17,15 +17,35 @@
  */
 package org.apache.hadoop.hdds;
 
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 /**
- * Config class for HDDS.
+ * This class contains constants for configuration keys and default values
+ * used in hdds.
  */
 public final class HddsConfigKeys {
+
+  /**
+   * Do not instantiate.
+   */
   private HddsConfigKeys() {
   }
+
+  public static final String HDDS_HEARTBEAT_INTERVAL =
+      "hdds.heartbeat.interval";
+  public static final String HDDS_HEARTBEAT_INTERVAL_DEFAULT =
+      "30s";
+
+  public static final String HDDS_NODE_REPORT_INTERVAL =
+      "hdds.node.report.interval";
+  public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT =
+      "60s";
+
+  public static final String HDDS_CONTAINER_REPORT_INTERVAL =
+      "hdds.container.report.interval";
+  public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
+      "60s";
+
   public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
       "hdds.command.status.report.interval";
   public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
-      ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT;
+      "60s";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 46eb8aa..71184cf 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -156,11 +156,6 @@ public final class ScmConfigKeys {
       "ozone.scm.handler.count.key";
   public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
 
-  public static final String OZONE_SCM_HEARTBEAT_INTERVAL =
-      "ozone.scm.heartbeat.interval";
-  public static final String OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT =
-      "30s";
-
   public static final String OZONE_SCM_DEADNODE_INTERVAL =
       "ozone.scm.dead.node.interval";
   public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index fc10fbb..0273677 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -200,11 +200,6 @@ public final class OzoneConfigKeys {
   public static final int
       OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10;
 
-  public static final String OZONE_CONTAINER_REPORT_INTERVAL =
-      "ozone.container.report.interval";
-  public static final String OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT =
-      "60s";
-
   public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
       = ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
   public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index da3870e..5a1d26a 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -153,13 +153,29 @@
     <description>The timeout duration for ratis server request.</description>
   </property>
   <property>
-    <name>ozone.container.report.interval</name>
+    <name>hdds.node.report.interval</name>
+    <value>60000ms</value>
+    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+    <description>Time interval of the datanode to send node report. Each
+      datanode periodically send node report to SCM. Unit could be
+      defined with postfix (ns,ms,s,m,h,d)</description>
+  </property>
+  <property>
+    <name>hdds.container.report.interval</name>
     <value>60000ms</value>
     <tag>OZONE, CONTAINER, MANAGEMENT</tag>
     <description>Time interval of the datanode to send container report. Each
-      datanode periodically send container report upon receive
-      sendContainerReport from SCM. Unit could be defined with
-      postfix (ns,ms,s,m,h,d)</description>
+      datanode periodically send container report to SCM. Unit could be
+      defined with postfix (ns,ms,s,m,h,d)</description>
+  </property>
+  <property>
+    <name>hdds.command.status.report.interval</name>
+    <value>60000ms</value>
+    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+    <description>Time interval of the datanode to send status of command
+      execution. Each datanode periodically the execution status of commands
+      received from SCM to SCM. Unit could be defined with postfix
+      (ns,ms,s,m,h,d)</description>
   </property>
   <!--Ozone Settings-->
   <property>
@@ -677,7 +693,7 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.heartbeat.interval</name>
+    <name>hdds.heartbeat.interval</name>
     <value>30s</value>
     <tag>OZONE, MANAGEMENT</tag>
     <description>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index cc7adbf..a8b919d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -29,13 +29,15 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_HEARTBEAT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_HEARTBEAT_INTERVAL_DEFAULT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_DEADNODE_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT;
@@ -181,9 +183,8 @@ public final class HddsServerUtil {
    * @return - HB interval in seconds.
    */
   public static long getScmHeartbeatInterval(Configuration conf) {
-    return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL,
-        ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT,
-        TimeUnit.SECONDS);
+    return conf.getTimeDuration(HDDS_HEARTBEAT_INTERVAL,
+        HDDS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
   }
 
   /**
@@ -225,7 +226,7 @@ public final class HddsServerUtil {
       sanitizeUserArgs(staleNodeIntervalMs, heartbeatIntervalMs, 3, 1000);
     } catch (IllegalArgumentException ex) {
       LOG.error("Stale Node Interval MS is cannot be honored due to " +
-          "mis-configured {}. ex:  {}", OZONE_SCM_HEARTBEAT_INTERVAL, ex);
+          "mis-configured {}. ex:  {}", HDDS_HEARTBEAT_INTERVAL, ex);
       throw ex;
     }
     return staleNodeIntervalMs;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
index ca5174a..3898d15 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
@@ -19,12 +19,20 @@ package org.apache.hadoop.ozone.container.common.report;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
 import org.apache.hadoop.hdds.protocol.proto.
     StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
 import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_COMMAND_STATUS_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT;
+
 /**
  * Publishes CommandStatusReport which will be sent to SCM as part of
  * heartbeat. CommandStatusReport consist of the following information:
@@ -42,9 +50,17 @@ public class CommandStatusReportPublisher extends
   protected long getReportFrequency() {
     if (cmdStatusReportInterval == -1) {
       cmdStatusReportInterval = getConf().getTimeDuration(
-          HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL,
-          HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT,
+          HDDS_COMMAND_STATUS_REPORT_INTERVAL,
+          HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT,
           TimeUnit.MILLISECONDS);
+
+      long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+          getConf());
+
+      Preconditions.checkState(
+          heartbeatFrequency < cmdStatusReportInterval,
+          HDDS_COMMAND_STATUS_REPORT_INTERVAL +
+              " cannot be configured lower than heartbeat frequency.");
     }
     return cmdStatusReportInterval;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
index ea2b987..3e73bb4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java
@@ -17,13 +17,20 @@
 
 package org.apache.hadoop.ozone.container.common.report;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
 
+import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT;
+
 
 /**
  * Publishes ContainerReport which will be sent to SCM as part of heartbeat.
@@ -49,9 +56,17 @@ public class ContainerReportPublisher extends
   protected long getReportFrequency() {
     if (containerReportInterval == null) {
       containerReportInterval = getConf().getTimeDuration(
-          OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL,
-          OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT,
+          HDDS_CONTAINER_REPORT_INTERVAL,
+          HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT,
           TimeUnit.MILLISECONDS);
+
+      long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+          getConf());
+
+      Preconditions.checkState(
+          heartbeatFrequency < containerReportInterval,
+          HDDS_CONTAINER_REPORT_INTERVAL +
+              " cannot be configured lower than heartbeat frequency.");
     }
     // Add a random delay (0~30s) on top of the container report
     // interval (60s) so tha the SCM is overwhelmed by the container reports
@@ -64,7 +79,7 @@ public class ContainerReportPublisher extends
   }
 
   @Override
-  protected ContainerReportsProto getReport() {
-    return ContainerReportsProto.getDefaultInstance();
+  protected ContainerReportsProto getReport() throws IOException {
+    return getContext().getParent().getContainer().getContainerReport();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
index 704b1f5..b98cba4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java
@@ -17,8 +17,18 @@
 
 package org.apache.hadoop.ozone.container.common.report;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_NODE_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_NODE_REPORT_INTERVAL_DEFAULT;
 
 /**
  * Publishes NodeReport which will be sent to SCM as part of heartbeat.
@@ -28,13 +38,29 @@ import org.apache.hadoop.hdds.protocol.proto
  */
 public class NodeReportPublisher extends ReportPublisher<NodeReportProto> {
 
+  private Long nodeReportInterval;
+
   @Override
   protected long getReportFrequency() {
-    return 90000L;
+    if (nodeReportInterval == null) {
+      nodeReportInterval = getConf().getTimeDuration(
+          HDDS_NODE_REPORT_INTERVAL,
+          HDDS_NODE_REPORT_INTERVAL_DEFAULT,
+          TimeUnit.MILLISECONDS);
+
+      long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+          getConf());
+
+      Preconditions.checkState(
+          heartbeatFrequency < nodeReportInterval,
+          HDDS_NODE_REPORT_INTERVAL +
+              " cannot be configured lower than heartbeat frequency.");
+    }
+    return nodeReportInterval;
   }
 
   @Override
-  protected NodeReportProto getReport() {
-    return NodeReportProto.getDefaultInstance();
+  protected NodeReportProto getReport() throws IOException {
+    return getContext().getParent().getContainer().getNodeReport();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
index 105f073..e3910db 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
@@ -23,7 +23,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ozone.container.common.statemachine
     .DatanodeStateMachine.DatanodeStates;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
@@ -34,6 +37,9 @@ import java.util.concurrent.TimeUnit;
 public abstract class ReportPublisher<T extends GeneratedMessage>
     implements Configurable, Runnable {
 
+  private static final Logger LOG = LoggerFactory.getLogger(
+      ReportPublisher.class);
+
   private Configuration config;
   private StateContext context;
   private ScheduledExecutorService executor;
@@ -76,7 +82,11 @@ public abstract class ReportPublisher<T extends GeneratedMessage>
    * Generates and publishes the report to datanode state context.
    */
   private void publishReport() {
-    context.addReport(getReport());
+    try {
+      context.addReport(getReport());
+    } catch (IOException e) {
+      LOG.error("Exception while publishing report.", e);
+    }
   }
 
   /**
@@ -91,7 +101,7 @@ public abstract class ReportPublisher<T extends GeneratedMessage>
    *
    * @return datanode report
    */
-  protected abstract T getReport();
+  protected abstract T getReport() throws IOException;
 
   /**
    * Returns {@link StateContext}.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index d4db55b..a0db2e8 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -180,14 +180,9 @@ public class TestReportPublisher {
 
   @Test
   public void testAddingReportToHeartbeat() {
-    Configuration conf = new OzoneConfiguration();
-    ReportPublisherFactory factory = new ReportPublisherFactory(conf);
-    ReportPublisher nodeReportPublisher = factory.getPublisherFor(
-        NodeReportProto.class);
-    ReportPublisher containerReportPubliser = factory.getPublisherFor(
-        ContainerReportsProto.class);
-    GeneratedMessage nodeReport = nodeReportPublisher.getReport();
-    GeneratedMessage containerReport = containerReportPubliser.getReport();
+    GeneratedMessage nodeReport = NodeReportProto.getDefaultInstance();
+    GeneratedMessage containerReport = ContainerReportsProto
+        .getDefaultInstance();
     SCMHeartbeatRequestProto.Builder heartbeatBuilder =
         SCMHeartbeatRequestProto.newBuilder();
     heartbeatBuilder.setDatanodeDetails(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
index eb591be..ba691ca 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
@@ -37,10 +37,10 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CONTAINER_REPORT_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT;
 
 /**
  * A class that manages closing of containers. This allows transition from a
@@ -75,8 +75,8 @@ public class ContainerCloser {
     this.threadRunCount = new AtomicInteger(0);
     this.isRunning = new AtomicBoolean(false);
     this.reportInterval = this.configuration.getTimeDuration(
-        OZONE_CONTAINER_REPORT_INTERVAL,
-        OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
+        HDDS_CONTAINER_REPORT_INTERVAL,
+        HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
     Preconditions.checkState(this.reportInterval > 0,
         "report interval has to be greater than 0");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
index 74238a7..f7863bc 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
@@ -44,6 +44,8 @@ import java.io.File;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys
+    .HDDS_CONTAINER_REPORT_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_CONTAINER_SIZE_DEFAULT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
@@ -52,8 +54,6 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent
     .CREATE;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent
     .CREATED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
-    .OZONE_CONTAINER_REPORT_INTERVAL;
 
 /**
  * Test class for Closing Container.
@@ -72,7 +72,7 @@ public class TestContainerCloser {
     configuration = SCMTestUtils.getConf();
     size = configuration.getLong(OZONE_SCM_CONTAINER_SIZE_GB,
         OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
-    configuration.setTimeDuration(OZONE_CONTAINER_REPORT_INTERVAL,
+    configuration.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL,
         1, TimeUnit.SECONDS);
     testDir = GenericTestUtils
         .getTestDir(TestContainerMapping.class.getSimpleName());
@@ -137,7 +137,7 @@ public class TestContainerCloser {
     // second report is discarded by the system if it lands in the 3 * report
     // frequency window.
 
-    configuration.setTimeDuration(OZONE_CONTAINER_REPORT_INTERVAL, 1,
+    configuration.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1,
         TimeUnit.SECONDS);
 
     ContainerWithPipeline containerWithPipeline = mapping.allocateContainer(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index cefd179..5275992 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -58,11 +58,10 @@ import java.util.concurrent.TimeoutException;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_DEADNODE_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_STALENODE_INTERVAL;
@@ -359,7 +358,7 @@ public class TestNodeManager {
     final int interval = 100;
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
         MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
 
     // This should be 5 times more than  OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL
     // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL
@@ -388,7 +387,7 @@ public class TestNodeManager {
     final int interval = 100;
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
         TimeUnit.MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS);
 
     // This should be 5 times more than  OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL
     // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL
@@ -413,7 +412,7 @@ public class TestNodeManager {
     OzoneConfiguration conf = getConf();
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
         MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
     conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
 
@@ -551,7 +550,7 @@ public class TestNodeManager {
     OzoneConfiguration conf = getConf();
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
         MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
     conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
 
@@ -729,7 +728,7 @@ public class TestNodeManager {
     OzoneConfiguration conf = getConf();
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
         MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
     conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
 
@@ -820,7 +819,7 @@ public class TestNodeManager {
     OzoneConfiguration conf = getConf();
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
         MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1,
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1,
         SECONDS);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000,
         MILLISECONDS);
@@ -985,7 +984,7 @@ public class TestNodeManager {
 
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
         MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
     conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 3ef74b0..9b7e399 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -59,6 +59,7 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
     .HEALTHY;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
@@ -392,11 +393,11 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster
{
 
     private void configureSCMheartbeat() {
       if (hbInterval.isPresent()) {
-        conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL,
+        conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL,
             hbInterval.get(), TimeUnit.MILLISECONDS);
 
       } else {
-        conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL,
+        conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL,
             DEFAULT_HB_INTERVAL_MS,
             TimeUnit.MILLISECONDS);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index cc367b3..7ca5fa1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.ozone;
 
-import static org.junit.Assert.fail;
 import java.io.IOException;
 
 import org.apache.commons.lang3.RandomUtils;
@@ -68,6 +67,9 @@ import org.junit.rules.Timeout;
 import org.mockito.Mockito;
 import org.apache.hadoop.test.GenericTestUtils;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static org.junit.Assert.fail;
+
 /**
  * Test class that exercises the StorageContainerManager.
  */
@@ -186,9 +188,7 @@ public class TestStorageContainerManager {
   public void testBlockDeletionTransactions() throws Exception {
     int numKeys = 5;
     OzoneConfiguration conf = new OzoneConfiguration();
-    conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL,
-        5,
-        TimeUnit.SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 5, TimeUnit.SECONDS);
     conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
         3000,
         TimeUnit.MILLISECONDS);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68b57ad3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
index 22528e4..a4fcd53 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
@@ -38,8 +39,6 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_DEADNODE_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_HEARTBEAT_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys
     .OZONE_SCM_STALENODE_INTERVAL;
@@ -61,7 +60,7 @@ public class TestQueryNode {
 
     conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
         interval, TimeUnit.MILLISECONDS);
-    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
+    conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
     conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message