hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject hadoop git commit: HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru.
Date Thu, 09 Mar 2017 22:34:31 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b6c477691 -> 5cdcda34a


HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cdcda34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cdcda34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cdcda34

Branch: refs/heads/branch-2
Commit: 5cdcda34a2e859862eb68bff5844c18ac4d27917
Parents: b6c4776
Author: Arpit Agarwal <arp@apache.org>
Authored: Thu Mar 9 14:34:10 2017 -0800
Committer: Arpit Agarwal <arp@apache.org>
Committed: Thu Mar 9 14:34:10 2017 -0800

----------------------------------------------------------------------
 .../hadoop-common/src/site/markdown/Metrics.md  |  7 +++++-
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  8 ++----
 .../apache/hadoop/hdfs/server/common/Util.java  | 26 +++++++++++++++++---
 .../hadoop/hdfs/server/datanode/DNConf.java     |  7 +++---
 .../hdfs/server/datanode/FileIoProvider.java    |  2 +-
 .../server/datanode/ProfilingFileIoEvents.java  | 11 ++++-----
 .../src/main/resources/hdfs-default.xml         |  2 +-
 .../datanode/TestDataNodeVolumeMetrics.java     |  4 +--
 .../hadoop/tools/TestHdfsConfigFields.java      |  2 --
 9 files changed, 43 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index e52a6a7..020d1ad 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -323,7 +323,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional
i
 FsVolume
 --------
 
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are
off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to
**true**, but enabling per-volume metrics may have a performance impact. Each metrics record
contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
+metrics are off by default. They can be enabled by setting `dfs.datanode
+.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
+Setting this value to 0.0 would mean profiling is not enabled. But enabling
+per-volume metrics may have a performance impact. Each metrics record
+contains tags such as Hostname as additional information along with metrics.
 
 | Name | Description |
 |:---- |:---- |

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e9e6bcc..4732359 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -634,7 +634,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int     DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT = 10000;
 
   public static final String DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY =
-      "dfs.datanode.slow.peers.report.interval";
+      "dfs.datanode.outliers.report.interval";
   public static final int DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT =
       1800 * 1000;
 
@@ -664,10 +664,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024;
 
   // Datanode File IO Stats
-  public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
-      "dfs.datanode.enable.fileio.profiling";
-  public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
-      false;
   public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
       "dfs.datanode.enable.fileio.fault.injection";
   public static final boolean
@@ -676,7 +672,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
       "dfs.datanode.fileio.profiling.sampling.fraction";
   public static final double
-      DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
+      DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
 
   //Keys with no defaults
   public static final String  DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index 6ec686f..07ad118 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -17,6 +17,11 @@
  */
 package org.apache.hadoop.hdfs.server.common;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
@@ -25,10 +30,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-
 @InterfaceAudience.Private
 public final class Util {
   private final static Log LOG = LogFactory.getLog(Util.class.getName());
@@ -102,4 +103,21 @@ public final class Util {
     }
     return uris;
   }
+
+  public static boolean isDiskStatsEnabled(double fileIOSamplingFraction) {
+    final boolean isEnabled;
+    if (fileIOSamplingFraction < 0.000001) {
+      LOG.info(DFSConfigKeys
+          .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+          + fileIOSamplingFraction + ". Disabling file IO profiling");
+      isEnabled = false;
+    } else {
+      LOG.info(DFSConfigKeys
+          .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
+          + fileIOSamplingFraction + ". Enabling file IO profiling");
+      isEnabled = true;
+    }
+
+    return isEnabled;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index c98793c..de74681 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
 import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
+import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.security.SaslPropertiesResolver;
 
 import java.util.concurrent.TimeUnit;
@@ -171,9 +172,9 @@ public class DNConf {
     this.peerStatsEnabled = getConf().getBoolean(
         DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
         DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
-    this.diskStatsEnabled = getConf().getBoolean(
-        DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY,
-        DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+    this.diskStatsEnabled = Util.isDiskStatsEnabled(getConf().getDouble(
+        DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+        DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT));
     this.outliersReportIntervalMs = getConf().getTimeDuration(
         DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
         DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 108088d..5986b88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
  *
  * Behavior can be injected into these events by enabling the
  * profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
  * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
  * These event hooks are disabled by default.
  *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 61d0432..2990105 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.util.Time;
@@ -43,13 +44,11 @@ class ProfilingFileIoEvents {
 
   public ProfilingFileIoEvents(@Nullable Configuration conf) {
     if (conf != null) {
-      isEnabled = conf.getBoolean(DFSConfigKeys
-          .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
-          .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
-      double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
-              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+      double fileIOSamplingFraction = conf.getDouble(
+          DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
           DFSConfigKeys
-              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
+      isEnabled = Util.isDiskStatsEnabled(fileIOSamplingFraction);
       if (fileIOSamplingFraction > 1) {
         LOG.warn(DFSConfigKeys
             .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index ca08eb3..f9c691a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1931,7 +1931,7 @@
 
 <property>
   <name>dfs.datanode.outliers.report.interval</name>
-  <value>30m</value>
+  <value>1800000</value>
   <description>
     This setting controls how frequently DataNodes will report their peer
     latencies to the NameNode via heartbeats.  This setting supports

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 6a8ac9c..03e1dee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
 
   private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
     Configuration conf = new HdfsConfiguration();
-    conf.setBoolean(DFSConfigKeys
-        .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
+    conf.setDouble(DFSConfigKeys
+        .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
     SimulatedFSDataset.setFactory(conf);
     return new MiniDFSCluster.Builder(conf)
         .numDataNodes(NUM_DATANODES)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cdcda34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index da5ef4d..6b6c87a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -107,8 +107,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase
{
         .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
     configurationPropsToSkipCompare
         .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
-    configurationPropsToSkipCompare
-        .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
     configurationPropsToSkipCompare.add(DFSConfigKeys
         .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
     configurationPropsToSkipCompare.add(DFSConfigKeys


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message