hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [07/34] hadoop git commit: HDFS-9521. TransferFsImage.receiveFile should account and log separate times for image download and fsync to disk. Contributed by Wellington Chevreuil
Date Wed, 09 Mar 2016 17:45:07 GMT
HDFS-9521. TransferFsImage.receiveFile should account and log separate times for image download
and fsync to disk. Contributed by Wellington Chevreuil


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd1c09be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd1c09be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd1c09be

Branch: refs/heads/HDFS-1312
Commit: fd1c09be3e7c67c188a1dd7e4fccb3d92dcc5b5b
Parents: 8ed2e06
Author: Harsh J <harsh@cloudera.com>
Authored: Mon Mar 7 13:49:47 2016 +0530
Committer: Harsh J <harsh@cloudera.com>
Committed: Mon Mar 7 17:14:51 2016 +0530

----------------------------------------------------------------------
 .../hdfs/server/namenode/TransferFsImage.java   | 33 ++++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd1c09be/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index eda6303..0186d8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -30,6 +30,7 @@ import java.net.URL;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -481,6 +482,9 @@ public class TransferFsImage {
       MD5Hash advertisedDigest, String fsImageName, InputStream stream,
       DataTransferThrottler throttler) throws IOException {
     long startTime = Time.monotonicNow();
+    Map<FileOutputStream, File> streamPathMap = new HashMap<>();
+    StringBuilder xferStats = new StringBuilder();
+    double xferCombined = 0;
     if (localPaths != null) {
       // If the local paths refer to directories, use the server-provided header
       // as the filename within that directory
@@ -517,7 +521,9 @@ public class TransferFsImage {
               LOG.warn("Overwriting existing file " + f
                   + " with file downloaded from " + url);
             }
-            outputStreams.add(new FileOutputStream(f));
+            FileOutputStream fos = new FileOutputStream(f);
+            outputStreams.add(fos);
+            streamPathMap.put(fos, f);
           } catch (IOException ioe) {
             LOG.warn("Unable to download file " + f, ioe);
             // This will be null if we're downloading the fsimage to a file
@@ -550,11 +556,26 @@ public class TransferFsImage {
         }
       }
       finishedReceiving = true;
+      double xferSec = Math.max(
+                 ((float)(Time.monotonicNow() - startTime)) / 1000.0, 0.001);
+      long xferKb = received / 1024;
+      xferCombined += xferSec;
+      xferStats.append(
+          String.format(" The fsimage download took %.2fs at %.2f KB/s.",
+              xferSec, xferKb / xferSec));
     } finally {
       stream.close();
       for (FileOutputStream fos : outputStreams) {
+        long flushStartTime = Time.monotonicNow();
         fos.getChannel().force(true);
         fos.close();
+        double writeSec = Math.max(((float)
+               (flushStartTime - Time.monotonicNow())) / 1000.0, 0.001);
+        xferCombined += writeSec;
+        xferStats.append(String
+                .format(" Synchronous (fsync) write to disk of " +
+                 streamPathMap.get(fos).getAbsolutePath() +
+                " took %.2fs.", writeSec));
       }
 
       // Something went wrong and did not finish reading.
@@ -573,11 +594,11 @@ public class TransferFsImage {
                               advertisedSize);
       }
     }
-    double xferSec = Math.max(
-        ((float)(Time.monotonicNow() - startTime)) / 1000.0, 0.001);
-    long xferKb = received / 1024;
-    LOG.info(String.format("Transfer took %.2fs at %.2f KB/s",
-        xferSec, xferKb / xferSec));
+    xferStats.insert(
+        0, String.format(
+            "Combined time for fsimage download and fsync " +
+            "to all disks took %.2fs.", xferCombined));
+    LOG.info(xferStats.toString());
 
     if (digester != null) {
       MD5Hash computedDigest = new MD5Hash(digester.digest());


Mime
View raw message