hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rake...@apache.org
Subject [41/50] [abbrv] hadoop git commit: HDFS-11338: [SPS]: Fix timeout issue in unit tests caused by longger NN down time. Contributed by Wei Zhou and Rakesh R
Date Tue, 11 Jul 2017 16:25:27 GMT
HDFS-11338: [SPS]: Fix timeout issue in unit tests caused by longger NN down time. Contributed
by Wei Zhou and Rakesh R


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddea4648
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddea4648
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddea4648

Branch: refs/heads/HDFS-10285
Commit: ddea4648065a55de9706f9b68b77dab394c77e83
Parents: 8da2dc7
Author: Uma Maheswara Rao G <uma.gangumalla@intel.com>
Authored: Tue Apr 11 14:25:01 2017 -0700
Committer: Rakesh Radhakrishnan <rakeshr@apache.org>
Committed: Tue Jul 11 19:35:11 2017 +0530

----------------------------------------------------------------------
 .../server/blockmanagement/BlockManager.java    | 13 +++++--
 .../BlockStorageMovementAttemptedItems.java     | 25 +++++++++----
 .../hdfs/server/namenode/FSNamesystem.java      |  2 +-
 .../server/namenode/StoragePolicySatisfier.java | 38 ++++++++++++++------
 .../TestBlockStorageMovementAttemptedItems.java |  3 +-
 5 files changed, 60 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddea4648/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 9d6b99c..6fe1cab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -707,7 +707,7 @@ public class BlockManager implements BlockStatsMXBean {
 
   public void close() {
     if (sps != null) {
-      sps.stop(false);
+      sps.deactivate(false);
     }
     bmSafeMode.close();
     try {
@@ -722,6 +722,7 @@ public class BlockManager implements BlockStatsMXBean {
     datanodeManager.close();
     pendingReconstruction.stop();
     blocksMap.close();
+    stopSPSGracefully();
   }
 
   /** @return the datanodeManager */
@@ -4900,10 +4901,18 @@ public class BlockManager implements BlockStatsMXBean {
       LOG.info("Storage policy satisfier is already stopped.");
       return;
     }
-    sps.stop(true);
+    sps.deactivate(true);
   }
 
   /**
+   * Timed wait to stop storage policy satisfier daemon threads.
+   */
+  public void stopSPSGracefully() {
+    if (sps != null) {
+      sps.stopGracefully();
+    }
+  }
+  /**
    * @return True if storage policy satisfier running.
    */
   public boolean isStoragePolicySatisfierRunning() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddea4648/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
index f15db73..26b98d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
@@ -130,21 +130,34 @@ public class BlockStorageMovementAttemptedItems {
   }
 
   /**
-   * Stops the monitor thread.
+   * Sets running flag to false. Also, this will interrupt monitor thread and
+   * clear all the queued up tasks.
    */
-  public synchronized void stop() {
+  public synchronized void deactivate() {
     monitorRunning = false;
     if (timerThread != null) {
       timerThread.interrupt();
-      try {
-        timerThread.join(3000);
-      } catch (InterruptedException ie) {
-      }
     }
     this.clearQueues();
   }
 
   /**
+   * Timed wait to stop monitor thread.
+   */
+  synchronized void stopGracefully() {
+    if (timerThread == null) {
+      return;
+    }
+    if (monitorRunning) {
+      deactivate();
+    }
+    try {
+      timerThread.join(3000);
+    } catch (InterruptedException ie) {
+    }
+  }
+
+  /**
    * This class contains information of an attempted trackID. Information such
    * as, (a)last attempted time stamp, (b)whether all the blocks in the trackID
    * were attempted and blocks movement has been scheduled to satisfy storage

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddea4648/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index a48a71a..cc8592f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1300,7 +1300,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       if (blockManager != null) {
         blockManager.deactivateSPS();
       }
-
       stopSecretManager();
       leaseManager.stopMonitor();
       if (nnrmthread != null) {
@@ -1336,6 +1335,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
         // Don't want to keep replication queues when not in Active.
         blockManager.clearQueues();
         blockManager.setInitializedReplQueues(false);
+        blockManager.stopSPSGracefully();
       }
     } finally {
       writeUnlock("stopActiveServices");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddea4648/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index 337d5b5..8cf9920 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -115,22 +115,21 @@ public class StoragePolicySatisfier implements Runnable {
   }
 
   /**
-   * Stop storage policy satisfier demon thread.
+   * Deactivates storage policy satisfier by stopping its services.
    *
-   * @param reconfigStop
+   * @param reconfig
+   *          true represents deactivating SPS service as requested by admin,
+   *          false otherwise
    */
-  public synchronized void stop(boolean reconfigStop) {
+  public synchronized void deactivate(boolean reconfig) {
     isRunning = false;
     if (storagePolicySatisfierThread == null) {
       return;
     }
+
     storagePolicySatisfierThread.interrupt();
-    try {
-      storagePolicySatisfierThread.join(3000);
-    } catch (InterruptedException ie) {
-    }
-    this.storageMovementsMonitor.stop();
-    if (reconfigStop) {
+    this.storageMovementsMonitor.deactivate();
+    if (reconfig) {
       LOG.info("Stopping StoragePolicySatisfier, as admin requested to "
           + "deactivate it.");
       this.clearQueuesWithNotification();
@@ -141,6 +140,23 @@ public class StoragePolicySatisfier implements Runnable {
   }
 
   /**
+   * Timed wait to stop storage policy satisfier daemon threads.
+   */
+  public synchronized void stopGracefully() {
+    if (isRunning) {
+      deactivate(true);
+    }
+    this.storageMovementsMonitor.stopGracefully();
+    if (storagePolicySatisfierThread == null) {
+      return;
+    }
+    try {
+      storagePolicySatisfierThread.join(3000);
+    } catch (InterruptedException ie) {
+    }
+  }
+
+  /**
    * Check whether StoragePolicySatisfier is running.
    * @return true if running
    */
@@ -162,7 +178,7 @@ public class StoragePolicySatisfier implements Runnable {
       if (!isRunning) {
         // Stopping monitor thread and clearing queues as well
         this.clearQueues();
-        this.storageMovementsMonitor.stop();
+        this.storageMovementsMonitor.stopGracefully();
         LOG.error(
             "Stopping StoragePolicySatisfier thread " + "as Mover ID file "
                 + HdfsServerConstants.MOVER_ID_PATH.toString()
@@ -194,7 +210,7 @@ public class StoragePolicySatisfier implements Runnable {
           isRunning = false;
           // Stopping monitor thread and clearing queues as well
           this.clearQueues();
-          this.storageMovementsMonitor.stop();
+          this.storageMovementsMonitor.stopGracefully();
         }
         if (!namesystem.isRunning()) {
           LOG.info("Stopping StoragePolicySatisfier.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddea4648/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
index 95142d3..8c7d982 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
@@ -47,7 +47,8 @@ public class TestBlockStorageMovementAttemptedItems {
   @After
   public void teardown() {
     if (bsmAttemptedItems != null) {
-      bsmAttemptedItems.stop();
+      bsmAttemptedItems.deactivate();
+      bsmAttemptedItems.stopGracefully();
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message