hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [44/50] [abbrv] hadoop git commit: HDFS-14008. NN should log snapshotdiff report. Contributed by Pranay Singh.
Date Fri, 02 Nov 2018 09:29:52 GMT
HDFS-14008. NN should log snapshotdiff report. Contributed by Pranay Singh.

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d98b881e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d98b881e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d98b881e

Branch: refs/heads/HDFS-13891
Commit: d98b881e9ab826eb7b70485d0de2a41ab7345334
Parents: 2ab611d
Author: Pranay Singh <psingh@cloudera.com>
Authored: Thu Nov 1 17:25:11 2018 -0700
Committer: Wei-Chiu Chuang <weichiu@apache.org>
Committed: Thu Nov 1 17:26:00 2018 -0700

----------------------------------------------------------------------
 .../hdfs/protocol/SnapshotDiffReport.java       | 65 ++++++++++++++++++++
 .../hdfs/server/namenode/FSNamesystem.java      | 20 ++++++
 .../snapshot/DirectorySnapshottableFeature.java |  5 ++
 .../namenode/snapshot/SnapshotDiffInfo.java     | 50 ++++++++++++++-
 4 files changed, 139 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
index 8ee4ec7..7bc95c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
@@ -170,14 +170,75 @@ public class SnapshotDiffReport {
   /** end point of the diff */
   private final String toSnapshot;
 
+
   /** list of diff */
   private final List<DiffReportEntry> diffList;
 
+  /**
+   * Records the stats related to Snapshot diff operation.
+   */
+  public static class DiffStats {
+    // Total dirs processed
+    private long totalDirsProcessed;
+
+    // Total dirs compared
+    private long totalDirsCompared;
+
+    // Total files processed
+    private long totalFilesProcessed;
+
+    // Total files compared
+    private long totalFilesCompared;
+
+    // Total children listing time
+    private final long totalChildrenListingTime;
+
+    public DiffStats(long totalDirsProcessed, long totalDirsCompared,
+                      long totalFilesProcessed, long totalFilesCompared,
+                      long totalChildrenListingTime) {
+      this.totalDirsCompared = totalDirsProcessed;
+      this.totalDirsProcessed = totalDirsCompared;
+      this.totalFilesCompared = totalFilesProcessed;
+      this.totalFilesProcessed = totalFilesCompared;
+      this.totalChildrenListingTime = totalChildrenListingTime;
+    }
+
+    public long getTotalDirsProcessed() {
+      return this.totalDirsProcessed;
+    }
+
+    public long getTotalDirsCompared() {
+      return this.totalDirsCompared;
+    }
+
+    public long getTotalFilesProcessed() {
+      return this.totalFilesProcessed;
+    }
+
+    public long getTotalFilesCompared() {
+      return this.totalFilesCompared;
+    }
+
+    public long getTotalChildrenListingTime() {
+      return totalChildrenListingTime;
+    }
+  }
+
+  /* Stats associated with the SnapshotDiff Report. */
+  private final DiffStats diffStats;
+
   public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
       String toSnapshot, List<DiffReportEntry> entryList) {
+    this(snapshotRoot, fromSnapshot, toSnapshot, new DiffStats(0, 0, 0, 0, 0),
+        entryList);
+  }
+
+  public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
+      String toSnapshot, DiffStats dStat, List<DiffReportEntry> entryList) {
     this.snapshotRoot = snapshotRoot;
     this.fromSnapshot = fromSnapshot;
     this.toSnapshot = toSnapshot;
+    this.diffStats = dStat;
     this.diffList = entryList != null ? entryList : Collections
         .<DiffReportEntry> emptyList();
   }
@@ -197,6 +258,10 @@ public class SnapshotDiffReport {
     return toSnapshot;
   }
 
+  public DiffStats getStats() {
+    return this.diffStats;
+  }
+
   /** @return {@link #diffList} */
   public List<DiffReportEntry> getDiffList() {
     return diffList;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d1904fa..8b1cdf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
+import org.apache.hadoop.util.Time;
 import static org.apache.hadoop.util.Time.now;
 import static org.apache.hadoop.util.Time.monotonicNow;
 import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
@@ -6624,6 +6625,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    */
   SnapshotDiffReport getSnapshotDiffReport(String path,
       String fromSnapshot, String toSnapshot) throws IOException {
+    long begTime = Time.monotonicNow();
     final String operationName = "computeSnapshotDiff";
     SnapshotDiffReport diffs = null;
     checkOperation(OperationCategory.READ);
@@ -6634,6 +6636,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
         path : Snapshot.getSnapshotPath(path, toSnapshot);
     final FSPermissionChecker pc = getPermissionChecker();
     readLock();
+    long actualTime = Time.monotonicNow();
     try {
       checkOperation(OperationCategory.READ);
       diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, pc, snapshotManager,
@@ -6646,6 +6649,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     } finally {
       readUnlock(operationName);
     }
+
+    if (diffs != null) {
+      SnapshotDiffReport.DiffStats dstat = diffs.getStats();
+      LOG.info("SnapshotDiffReport '"
+          + fromSnapshot
+          + "' to '" + toSnapshot + "'. Total comparison dirs: "
+          + dstat.getTotalDirsCompared()
+          + "/" + dstat.getTotalDirsProcessed()
+          + ", files: "
+          + dstat.getTotalFilesCompared()
+          + "/" + dstat.getTotalFilesProcessed()
+          + ". Time snapChildrenListing: "
+          + dstat.getTotalChildrenListingTime() / 1000.0 + "s, actual: "
+          + ((Time.monotonicNow() - actualTime) / 1000.0) + "s, total: "
+          + ((Time.monotonicNow() - begTime) / 1000.0) + "s.");
+    }
+
     logAuditEvent(success, operationName, fromSnapshotRoot,
         toSnapshotRoot, null);
     return diffs;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index 15aa22a..073b88b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -389,9 +389,13 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
         if (change) {
           diffReport.addDirDiff(dir, relativePath, diff);
         }
+      } else {
+        diffReport.incrementDirsProcessed();
       }
+      long startTime = Time.monotonicNow();
       ReadOnlyList<INode> children = dir.getChildrenList(earlierSnapshot
           .getId());
+      diffReport.addChildrenListingTime(Time.monotonicNow() - startTime);
       for (INode child : children) {
         final byte[] name = child.getLocalNameBytes();
         boolean toProcess = !diff.containsDeleted(name);
@@ -418,6 +422,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
       if (change) {
         diffReport.addFileDiff(file, relativePath);
       }
+      diffReport.incrementFilesProcessed();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
index 4c74afd..ab6f415 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
@@ -125,6 +125,21 @@ class SnapshotDiffInfo {
   private final Map<Long, RenameEntry> renameMap =
       new HashMap<Long, RenameEntry>();
 
+  // Total directories compared
+  private long totalDirsCompared;
+
+  // Total directories
+  private long totalDirsProcessed;
+
+  // Total files compared
+  private long totalFilesCompared;
+
+  // Total files
+  private long totalFilesProcessed;
+
+  // Total children listing time
+  private long childrenListingTime;
+
   SnapshotDiffInfo(INodeDirectory snapshotRootDir,
       INodeDirectory snapshotDiffScopeDir, Snapshot start, Snapshot end) {
     Preconditions.checkArgument(snapshotRootDir.isSnapshottable() &&
@@ -133,6 +148,10 @@ class SnapshotDiffInfo {
     this.snapshotDiffScopeDir = snapshotDiffScopeDir;
     this.from = start;
     this.to = end;
+    this.totalDirsCompared = 0;
+    this.totalDirsProcessed = 0;
+    this.totalFilesCompared = 0;
+    this.totalFilesProcessed = 0;
   }
 
   /** Add a dir-diff pair */
@@ -164,6 +183,29 @@ class SnapshotDiffInfo {
     return to;
   }
 
+
+  void incrementDirsCompared() {
+    this.totalDirsCompared++;
+    incrementDirsProcessed();
+  }
+
+  void incrementDirsProcessed() {
+    this.totalDirsProcessed++;
+  }
+
+  void incrementFilesCompared() {
+    this.totalFilesCompared++;
+    incrementFilesProcessed();
+  }
+
+  void incrementFilesProcessed() {
+    this.totalFilesProcessed++;
+  }
+
+  public void addChildrenListingTime(long millis) {
+    this.childrenListingTime += millis;
+  }
+
   private RenameEntry getEntry(long inodeId) {
     RenameEntry entry = renameMap.get(inodeId);
     if (entry == null) {
@@ -203,9 +245,15 @@ class SnapshotDiffInfo {
         diffReportList.addAll(subList);
       }
     }
+
+    SnapshotDiffReport.DiffStats dStats = new SnapshotDiffReport.DiffStats(
+        this.totalDirsCompared, this.totalDirsProcessed,
+        this.totalFilesCompared, this.totalFilesProcessed,
+        this.childrenListingTime);
+
     return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
         Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
-        diffReportList);
+        dStats, diffReportList);
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message