Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: zhz@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <4540b0780d5f4a1dbe59f3a35eeecc24@git.apache.org>
Subject: hadoop git commit: HDFS-12502. nntop should support a category based
 on FilesInGetListingOps.
Date: Thu, 19 Oct 2017 07:08:32 +0000 (UTC)
archived-at: Thu, 19 Oct 2017 07:08:35 -0000

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e7e2b8158 -> 87f440739


HDFS-12502. nntop should support a category based on FilesInGetListingOps.

(cherry picked from commit 60bfee270ed3a653c44c0bc92396167b5022df6e)
(cherry picked from commit aecf6c50d351e7aed2008a682677a1edb104585d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f44073
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f44073
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f44073

Branch: refs/heads/branch-2
Commit: 87f440739604f97261da9624afb36920a6a4c28f
Parents: e7e2b81
Author: Zhe Zhang <zhz@apache.org>
Authored: Wed Oct 18 23:51:24 2017 -0700
Committer: Zhe Zhang <zhz@apache.org>
Committed: Thu Oct 19 00:05:34 2017 -0700

----------------------------------------------------------------------
 .../hdfs/server/namenode/FSNamesystem.java      |  5 ++++
 .../server/namenode/top/metrics/TopMetrics.java | 30 +++++++++++++++-----
 .../server/namenode/metrics/TestTopMetrics.java | 11 +++++--
 3 files changed, 36 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f44073/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e21da7f..eded007 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3619,6 +3619,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       readUnlock(operationName);
     }
     logAuditEvent(true, operationName, src);
+    if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation()
+        && dl != null && Server.getRemoteUser() != null) {
+      topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(),
+          dl.getPartialListing().length);
+    }
     return dl;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f44073/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
index 2719c88..3d8dd19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
@@ -70,6 +70,14 @@ public class TopMetrics implements MetricsSource {
   public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
   public static final String TOPMETRICS_METRICS_SOURCE_NAME =
       "NNTopUserOpCounts";
+  /**
+   * In addition to counts of different RPC calls, NNTop also reports top
+   * users listing large directories (measured by the number of files involved
+   * in listing operations from the user). This is important because the CPU
+   * and GC overhead of a listing operation grows linearly with the number of
+   * files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}.
+   */
+  public static final String FILES_IN_GETLISTING = "filesInGetListing";
   private final boolean isMetricsSourceEnabled;
 
   private static void logConf(Configuration conf) {
@@ -123,22 +131,30 @@ public class TopMetrics implements MetricsSource {
   public void report(boolean succeeded, String userName, InetAddress addr,
       String cmd, String src, String dst, FileStatus status) {
     // currently nntop only makes use of the username and the command
-    report(userName, cmd);
+    report(userName, cmd, 1);
   }
 
-  public void report(String userName, String cmd) {
+  public void reportFilesInGetListing(String userName, int numFiles) {
+    report(userName, FILES_IN_GETLISTING, numFiles);
+  }
+
+  public void report(String userName, String cmd, int delta) {
     long currTime = Time.monotonicNow();
-    report(currTime, userName, cmd);
+    report(currTime, userName, cmd, delta);
   }
 
-  public void report(long currTime, String userName, String cmd) {
+  public void report(long currTime, String userName, String cmd, int delta) {
     LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
     userName = UserGroupInformation.trimLoginMethod(userName);
     for (RollingWindowManager rollingWindowManager : rollingWindowManagers
         .values()) {
-      rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
-      rollingWindowManager.recordMetric(currTime,
-          TopConf.ALL_CMDS, userName, 1);
+      rollingWindowManager.recordMetric(currTime, cmd, userName, delta);
+      // Increase the number of all RPC calls by the user, unless the report
+      // is for the number of files in a listing operation.
+      if (!cmd.equals(FILES_IN_GETLISTING)) {
+        rollingWindowManager.recordMetric(currTime,
+            TopConf.ALL_CMDS, userName, delta);
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f44073/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
index 4d3a4f0..cda4b14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.metrics2.lib.Interns;
 import org.junit.Test;
 
 import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
+import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.FILES_IN_GETLISTING;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -41,9 +42,10 @@ public class TestTopMetrics {
     TopMetrics topMetrics = new TopMetrics(conf,
         topConf.nntopReportingPeriodsMs);
     // Dummy command
-    topMetrics.report("test", "listStatus");
-    topMetrics.report("test", "listStatus");
-    topMetrics.report("test", "listStatus");
+    topMetrics.report("test", "listStatus", 1);
+    topMetrics.report("test", "listStatus", 1);
+    topMetrics.report("test", "listStatus", 1);
+    topMetrics.report("test", FILES_IN_GETLISTING, 1000);
 
     MetricsRecordBuilder rb = getMetrics(topMetrics);
     MetricsCollector mc = rb.parent();
@@ -59,5 +61,8 @@ public class TestTopMetrics {
 
     verify(rb, times(3)).addCounter(Interns.info("op=listStatus." +
         "user=test.count", "Total operations performed by user"), 3L);
+
+    verify(rb, times(3)).addCounter(Interns.info("op=" + FILES_IN_GETLISTING +
+        ".user=test.count", "Total operations performed by user"), 1000L);
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org