hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [hadoop] branch branch-3.2 updated: HDFS-13694. Making md5 computing being in parallel with image loading.
Date Mon, 08 Jul 2019 03:05:17 GMT
This is an automated email from the ASF dual-hosted git repository.

inigoiri pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new adec307  HDFS-13694. Making md5 computing being in parallel with image loading.
adec307 is described below

commit adec307251118e8c1f5a00be58e125721a955e36
Author: leosunli <lisheng.sun08@gmail.com>
AuthorDate: Sat Jul 6 01:17:37 2019 +0800

    HDFS-13694. Making md5 computing being in parallel with image loading.
    
    
    (cherry picked from commit 9c907294860a48f0d1676a31bda15795b7a6771a)
---
 .../server/namenode/FSImageFormatProtobuf.java     | 65 +++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 7aed5fd..a00cde5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -172,13 +172,76 @@ public final class FSImageFormatProtobuf {
       return ctx;
     }
 
+    /**
+     * Thread to compute the MD5 of a file as this can be in parallel while
+     * loading the image without interfering much.
+     */
+    private static class DigestThread extends Thread {
+
+      /**
+       * Exception thrown when computing the digest if it cannot be calculated.
+       */
+      private volatile IOException ioe = null;
+
+      /**
+       * Calculated digest if there are no error.
+       */
+      private volatile MD5Hash digest = null;
+
+      /**
+       * FsImage file computed MD5.
+       */
+      private final File file;
+
+      DigestThread(File inFile) {
+        file = inFile;
+        setName(inFile.getName() + " MD5 compute");
+        setDaemon(true);
+      }
+
+      public MD5Hash getDigest() throws IOException {
+        if (ioe != null) {
+          throw ioe;
+        }
+        return digest;
+      }
+
+      public IOException getException() {
+        return ioe;
+      }
+
+      @Override
+      public void run() {
+        try {
+          digest = MD5FileUtils.computeMd5ForFile(file);
+        } catch (IOException e) {
+          ioe = e;
+        } catch (Throwable t) {
+          ioe = new IOException(t);
+        }
+      }
+
+      @Override
+      public String toString() {
+        return "DigestThread{ ThreadName=" + getName() + ", digest=" + digest
+            + ", file=" + file + '}';
+      }
+    }
+
     void load(File file) throws IOException {
       long start = Time.monotonicNow();
-      imgDigest = MD5FileUtils.computeMd5ForFile(file);
+      DigestThread dt = new DigestThread(file);
+      dt.start();
       RandomAccessFile raFile = new RandomAccessFile(file, "r");
       FileInputStream fin = new FileInputStream(file);
       try {
         loadInternal(raFile, fin);
+        try {
+          dt.join();
+          imgDigest = dt.getDigest();
+        } catch (InterruptedException ie) {
+          throw new IOException(ie);
+        }
         long end = Time.monotonicNow();
         LOG.info("Loaded FSImage in {} seconds.", (end - start) / 1000);
       } finally {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message