hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [hadoop-ozone] branch ozone-0.5.0 updated: HDDS-3072. SCM scrub pipeline should be started after coming out of safe mode. (#605)
Date Fri, 06 Mar 2020 17:00:45 GMT
This is an automated email from the ASF dual-hosted git repository.

arp pushed a commit to branch ozone-0.5.0
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/ozone-0.5.0 by this push:
     new 9220252  HDDS-3072. SCM scrub pipeline should be started after coming out of safe
mode. (#605)
9220252 is described below

commit 92202523bd2a82b2d5dd6ebe80b7a0f9c95db902
Author: Bharat Viswanadham <bharat@apache.org>
AuthorDate: Fri Mar 6 07:53:16 2020 -0800

    HDDS-3072. SCM scrub pipeline should be started after coming out of safe mode. (#605)
    
    
    (cherry picked from commit 252f56d63d157eeb8b0da05e5fbb535783468b99)
---
 .../scm/pipeline/BackgroundPipelineCreator.java    | 10 +++++----
 .../hadoop/hdds/scm/pipeline/PipelineManager.java  | 13 +++++++++++
 .../hdds/scm/pipeline/SCMPipelineManager.java      | 19 +++++++++++++++-
 .../hdds/scm/safemode/SCMSafeModeManager.java      |  1 -
 .../hadoop/hdds/scm/safemode/SafeModeHandler.java  | 25 ++++------------------
 .../hdds/scm/server/StorageContainerManager.java   |  3 +++
 6 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index 8e4ec6a..b8f0fb6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -111,10 +111,12 @@ class BackgroundPipelineCreator {
         continue;
       }
 
-      try {
-        pipelineManager.scrubPipeline(type, factor);
-      } catch (IOException e) {
-        LOG.error("Error while scrubbing pipelines {}", e);
+      if (!pipelineManager.getSafeModeStatus()) {
+        try {
+          pipelineManager.scrubPipeline(type, factor);
+        } catch (IOException e) {
+          LOG.error("Error while scrubbing pipelines {}", e);
+        }
       }
 
       while (true) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
index 635e032..68d66ff 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
@@ -112,4 +112,17 @@ public interface PipelineManager extends Closeable, PipelineManagerMXBean
{
   default void waitPipelineReady(PipelineID pipelineID, long timeout)
       throws IOException {
   }
+
+  /**
+   * Set SafeMode status.
+   *
+   * @param safeModeStatus
+   */
+  void setSafeModeStatus(boolean safeModeStatus);
+
+  /**
+   * Get SafeMode status.
+   * @return boolean
+   */
+  boolean getSafeModeStatus();
 }
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index eb2e0d6..0989d34 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -54,6 +54,7 @@ import java.util.NavigableSet;
 import java.util.Set;
 import java.util.Collection;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
@@ -85,6 +86,8 @@ public class SCMPipelineManager implements PipelineManager {
   // Pipeline Manager MXBean
   private ObjectName pmInfoBean;
 
+  private final AtomicBoolean isInSafeMode;
+
   public SCMPipelineManager(Configuration conf, NodeManager nodeManager,
       EventPublisher eventPublisher)
       throws IOException {
@@ -127,6 +130,9 @@ public class SCMPipelineManager implements PipelineManager {
         HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL,
         HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT,
         TimeUnit.MILLISECONDS);
+    this.isInSafeMode = new AtomicBoolean(conf.getBoolean(
+        HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED,
+        HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED_DEFAULT));
   }
 
   public PipelineStateManager getStateManager() {
@@ -414,7 +420,7 @@ public class SCMPipelineManager implements PipelineManager {
             .toEpochMilli() >= pipelineScrubTimeoutInMills)
         .collect(Collectors.toList());
     for (Pipeline p : needToSrubPipelines) {
-      LOG.info("srubbing pipeline: id: " + p.getId().toString() +
+      LOG.info("Scrubbing pipeline: id: " + p.getId().toString() +
           " since it stays at ALLOCATED stage for " +
           Duration.between(currentTime, p.getCreationTimestamp()).toMinutes() +
           " mins.");
@@ -618,4 +624,15 @@ public class SCMPipelineManager implements PipelineManager {
   protected NodeManager getNodeManager() {
     return nodeManager;
   }
+
+  @Override
+  public void setSafeModeStatus(boolean safeModeStatus) {
+    this.isInSafeMode.set(safeModeStatus);
+  }
+
+  @Override
+  public boolean getSafeModeStatus() {
+    return this.isInSafeMode.get();
+  }
+
 }
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
index 90fce3f..ca484c8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
@@ -134,7 +134,6 @@ public class SCMSafeModeManager implements SafeModeManager {
         exitRules.put(ATLEAST_ONE_DATANODE_REPORTED_PIPELINE_EXIT_RULE,
             oneReplicaPipelineSafeModeRule);
       }
-      emitSafeModeStatus();
       boolean createPipelineInSafemode = conf.getBoolean(
           HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION,
           HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION_DEFAULT);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
index 2fbe893..50095ab 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.scm.block.BlockManager;
 import org.apache.hadoop.hdds.scm.container.ReplicationManager;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
 import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
 import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
@@ -30,8 +29,6 @@ import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -94,8 +91,8 @@ public class SafeModeHandler implements EventHandler<SafeModeStatus>
{
    * Set SafeMode status based on
    * {@link org.apache.hadoop.hdds.scm.events.SCMEvents#SAFE_MODE_STATUS}.
    *
-   * Inform BlockManager, ScmClientProtocolServer and replicationAcitivity
-   * status about safeMode status.
+   * Inform BlockManager, ScmClientProtocolServer, ScmPipeline Manager and
+   * Replication Manager status about safeMode status.
    *
    * @param safeModeStatus
    * @param publisher
@@ -114,8 +111,9 @@ public class SafeModeHandler implements EventHandler<SafeModeStatus>
{
         } catch (InterruptedException e) {
           Thread.currentThread().interrupt();
         }
+        scmPipelineManager.setSafeModeStatus(isInSafeMode.get());
         replicationManager.start();
-        cleanupPipelines();
+        scmPipelineManager.triggerPipelineCreation();
       });
 
       safeModeExitThread.setDaemon(true);
@@ -124,21 +122,6 @@ public class SafeModeHandler implements EventHandler<SafeModeStatus>
{
 
   }
 
-  private void cleanupPipelines() {
-    List<Pipeline> pipelineList = scmPipelineManager.getPipelines();
-    pipelineList.forEach((pipeline) -> {
-      try {
-        if (pipeline.getPipelineState() == Pipeline.PipelineState.ALLOCATED &&
-            pipeline.isAllocationTimeout()) {
-          scmPipelineManager.finalizeAndDestroyPipeline(pipeline, false);
-        }
-      } catch (IOException ex) {
-        LOG.error("Finalize and destroy pipeline failed for pipeline "
-            + pipeline.toString(), ex);
-      }
-    });
-  }
-
   public boolean getSafeModeStatus() {
     return isInSafeMode.get();
   }
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 6063a62..1334293 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -358,6 +358,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler);
     eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
     eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, safeModeHandler);
+
+    // Emit initial safe mode status, as now handlers are registered.
+    scmSafeModeManager.emitSafeModeStatus();
     registerMXBean();
     registerMetricsSource(this);
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message