hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ayushsax...@apache.org
Subject [hadoop] branch branch-3.3 updated: HDFS-15369. Refactor method VolumeScanner#runLoop(). Contributed by Yang Yun.
Date Sun, 24 May 2020 12:38:22 GMT
This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
     new 718c8a5  HDFS-15369. Refactor method VolumeScanner#runLoop(). Contributed by Yang
Yun.
718c8a5 is described below

commit 718c8a5868680115c0f5f8253a8c9f72e0069082
Author: Ayush Saxena <ayushsaxena@apache.org>
AuthorDate: Sun May 24 18:03:41 2020 +0530

    HDFS-15369. Refactor method VolumeScanner#runLoop(). Contributed by Yang Yun.
---
 .../hadoop/hdfs/server/datanode/VolumeScanner.java | 81 +++++++++++++---------
 1 file changed, 47 insertions(+), 34 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 5f1a1e0..5e3d523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -484,6 +484,50 @@ public class VolumeScanner extends Thread {
   }
 
   /**
+   * Get next block and check if it's needed to scan.
+   *
+   * @return  the candidate block.
+   */
+  ExtendedBlock getNextBlockToScan() {
+    ExtendedBlock block;
+    try {
+      block = curBlockIter.nextBlock();
+    } catch (IOException e) {
+      // There was an error listing the next block in the volume.  This is a
+      // serious issue.
+      LOG.warn("{}: nextBlock error on {}", this, curBlockIter);
+      // On the next loop iteration, curBlockIter#eof will be set to true, and
+      // we will pick a different block iterator.
+      return null;
+    }
+    if (block == null) {
+      // The BlockIterator is at EOF.
+      LOG.info("{}: finished scanning block pool {}",
+          this, curBlockIter.getBlockPoolId());
+      saveBlockIterator(curBlockIter);
+      return null;
+    } else if (conf.skipRecentAccessed) {
+      // Check the access time of block file to avoid scanning recently
+      // changed blocks, reducing disk IO.
+      try {
+        BlockLocalPathInfo blockLocalPathInfo =
+            volume.getDataset().getBlockLocalPathInfo(block);
+        BasicFileAttributes attr = Files.readAttributes(
+            new File(blockLocalPathInfo.getBlockPath()).toPath(),
+            BasicFileAttributes.class);
+        if (System.currentTimeMillis() - attr.lastAccessTime().
+            to(TimeUnit.MILLISECONDS) < conf.scanPeriodMs) {
+          return null;
+        }
+      } catch (IOException ioe) {
+        LOG.debug("Failed to get access time of block {}",
+            block, ioe);
+      }
+    }
+    return block;
+  }
+
+  /**
    * Run an iteration of the VolumeScanner loop.
    *
    * @param suspectBlock   A suspect block which we should scan, or null to
@@ -507,10 +551,10 @@ public class VolumeScanner extends Thread {
         return 30000L;
       }
 
-      // Find a usable block pool to scan.
       if (suspectBlock != null) {
         block = suspectBlock;
       } else {
+        // Find a usable block pool to scan.
         if ((curBlockIter == null) || curBlockIter.atEnd()) {
           long timeout = findNextUsableBlockIter();
           if (timeout > 0) {
@@ -528,40 +572,9 @@ public class VolumeScanner extends Thread {
           }
           return 0L;
         }
-        try {
-          block = curBlockIter.nextBlock();
-        } catch (IOException e) {
-          // There was an error listing the next block in the volume.  This is a
-          // serious issue.
-          LOG.warn("{}: nextBlock error on {}", this, curBlockIter);
-          // On the next loop iteration, curBlockIter#eof will be set to true, and
-          // we will pick a different block iterator.
-          return 0L;
-        }
+        block = getNextBlockToScan();
         if (block == null) {
-          // The BlockIterator is at EOF.
-          LOG.info("{}: finished scanning block pool {}",
-              this, curBlockIter.getBlockPoolId());
-          saveBlockIterator(curBlockIter);
-          return 0;
-        } else if (conf.skipRecentAccessed) {
-          // Check the access time of block file to avoid scanning recently
-          // changed blocks, reducing disk IO.
-          try {
-            BlockLocalPathInfo blockLocalPathInfo =
-                volume.getDataset().getBlockLocalPathInfo(block);
-            BasicFileAttributes attr = Files.readAttributes(
-                new File(blockLocalPathInfo.getBlockPath()).toPath(),
-                BasicFileAttributes.class);
-            if (System.currentTimeMillis() - attr.lastAccessTime().
-                to(TimeUnit.MILLISECONDS) < conf.scanPeriodMs) {
-              return 0;
-            }
-
-          } catch (IOException ioe) {
-            LOG.debug("Failed to get access time of block {}",
-                block, ioe);
-          }
+          return 0L;
         }
       }
       if (curBlockIter != null) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message