hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject hadoop git commit: HDFS-11339. Support File IO sampling for Datanode IO profiling hooks. Contributed by Hanisha Koneru.
Date Tue, 17 Jan 2017 06:52:16 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 52fdd387e -> be8e001f1


HDFS-11339. Support File IO sampling for Datanode IO profiling hooks. Contributed by Hanisha
Koneru.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be8e001f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be8e001f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be8e001f

Branch: refs/heads/branch-2
Commit: be8e001f1f1cf9c84e2c213bd619374d38020e5c
Parents: 52fdd38
Author: Arpit Agarwal <arp@apache.org>
Authored: Mon Jan 16 14:43:29 2017 -0800
Committer: Arpit Agarwal <arp@apache.org>
Committed: Mon Jan 16 22:49:14 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   | 16 +++++++++++----
 .../server/datanode/ProfilingFileIoEvents.java  | 21 ++++++++++++++++++--
 .../hadoop/tools/TestHdfsConfigFields.java      |  2 ++
 3 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be8e001f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index fb3acf7..d4ec567 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -642,10 +642,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY = "dfs.image.transfer.chunksize";
   public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024;
 
-  //Keys with no defaults
-  public static final String  DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
-  public static final String  DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory";
-  public static final String  DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy";
+  // Datanode File IO Stats
   public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
       "dfs.datanode.enable.fileio.profiling";
   public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
@@ -654,6 +651,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.datanode.enable.fileio.fault.injection";
   public static final boolean
       DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_DEFAULT = false;
+  public static final String
+      DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
+      "dfs.datanode.fileio.profiling.sampling.fraction";
+  public static final double
+      DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
+
+  //Keys with no defaults
+  public static final String  DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
+  public static final String  DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory";
+  public static final String  DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy";
+
   public static final String  DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY
= "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold";
   public static final long    DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT
= 1024L * 1024L * 1024L * 10L; // 10 GB
   public static final String  DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
= "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be8e001f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 43ac495..61d0432 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hdfs.server.datanode;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -26,6 +28,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.util.Time;
 
 import javax.annotation.Nullable;
+import java.util.concurrent.ThreadLocalRandom;
 
 /**
  * Profiles the performance of the metadata and data related operations on
@@ -33,16 +36,30 @@ import javax.annotation.Nullable;
  */
 @InterfaceAudience.Private
 class ProfilingFileIoEvents {
+  static final Log LOG = LogFactory.getLog(ProfilingFileIoEvents.class);
 
   private final boolean isEnabled;
+  private final int sampleRangeMax;
 
   public ProfilingFileIoEvents(@Nullable Configuration conf) {
     if (conf != null) {
       isEnabled = conf.getBoolean(DFSConfigKeys
           .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
           .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+      double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
+              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+          DFSConfigKeys
+              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+      if (fileIOSamplingFraction > 1) {
+        LOG.warn(DFSConfigKeys
+            .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
+            " value cannot be more than 1. Setting value to 1");
+        fileIOSamplingFraction = 1;
+      }
+      sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
     } else {
       isEnabled = false;
+      sampleRangeMax = 0;
     }
   }
 
@@ -69,7 +86,7 @@ class ProfilingFileIoEvents {
 
   public long beforeFileIo(@Nullable FsVolumeSpi volume,
       FileIoProvider.OPERATION op, long len) {
-    if (isEnabled) {
+    if (isEnabled && ThreadLocalRandom.current().nextInt() < sampleRangeMax) {
       DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
       if (metrics != null) {
         return Time.monotonicNow();
@@ -80,7 +97,7 @@ class ProfilingFileIoEvents {
 
   public void afterFileIo(@Nullable FsVolumeSpi volume,
       FileIoProvider.OPERATION op, long begin, long len) {
-    if (isEnabled) {
+    if (isEnabled && begin != 0) {
       DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
       if (metrics != null) {
         long latency = Time.monotonicNow() - begin;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be8e001f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 71ad713..da10878 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -107,6 +107,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase
{
         .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
     configurationPropsToSkipCompare.add(DFSConfigKeys
         .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
+    configurationPropsToSkipCompare.add(DFSConfigKeys
+        .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY);
 
     // Allocate
     xmlPropsToSkipCompare = new HashSet<String>();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message