hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: Revert "HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru."
Date Fri, 03 Mar 2017 00:22:58 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5e760f8dc -> d737a26c5


Revert "HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru."

This reverts commit 5e760f8dc4bc007bd839f33fe4a99fa603755149.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d737a26c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d737a26c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d737a26c

Branch: refs/heads/branch-2
Commit: d737a26c5a402918460e5c4f79c78b345fc604cd
Parents: 5e760f8
Author: Andrew Wang <wang@apache.org>
Authored: Thu Mar 2 16:21:32 2017 -0800
Committer: Andrew Wang <wang@apache.org>
Committed: Thu Mar 2 16:21:32 2017 -0800

----------------------------------------------------------------------
 .../hadoop-common/src/site/markdown/Metrics.md  |  7 +----
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  6 ++++-
 .../hdfs/server/datanode/FileIoProvider.java    |  2 +-
 .../server/datanode/ProfilingFileIoEvents.java  | 27 +++++---------------
 .../datanode/TestDataNodeVolumeMetrics.java     |  4 +--
 .../hadoop/tools/TestHdfsConfigFields.java      |  2 ++
 6 files changed, 17 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d737a26c/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 020d1ad..e52a6a7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -323,12 +323,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional
i
 FsVolume
 --------
 
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
-metrics are off by default. They can be enabled by setting `dfs.datanode
-.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
-Setting this value to 0.0 would mean profiling is not enabled. But enabling
-per-volume metrics may have a performance impact. Each metrics record
-contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are
off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to
**true**, but enabling per-volume metrics may have a performance impact. Each metrics record
contains tags such as Hostname as additional information along with metrics.
 
 | Name | Description |
 |:---- |:---- |

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d737a26c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 3b71c49..853306b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -664,6 +664,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024;
 
   // Datanode File IO Stats
+  public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
+      "dfs.datanode.enable.fileio.profiling";
+  public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
+      false;
   public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
       "dfs.datanode.enable.fileio.fault.injection";
   public static final boolean
@@ -672,7 +676,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
       "dfs.datanode.fileio.profiling.sampling.fraction";
   public static final double
-      DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
+      DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
 
   //Keys with no defaults
   public static final String  DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d737a26c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 5986b88..108088d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
  *
  * Behavior can be injected into these events by enabling the
  * profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
  * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
  * These event hooks are disabled by default.
  *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d737a26c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 580f704..61d0432 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hdfs.server.datanode;
 
-import static java.lang.Math.abs;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,32 +43,18 @@ class ProfilingFileIoEvents {
 
   public ProfilingFileIoEvents(@Nullable Configuration conf) {
     if (conf != null) {
+      isEnabled = conf.getBoolean(DFSConfigKeys
+          .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
+          .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
       double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
               .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
           DFSConfigKeys
-              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
-      if (abs(fileIOSamplingFraction) < 0.000001) {
-        LOG.info(DFSConfigKeys
-            .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
-            + fileIOSamplingFraction + ". Disabling file IO profiling");
-        isEnabled = false;
-      } else if (fileIOSamplingFraction < 0.000001) {
+              .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+      if (fileIOSamplingFraction > 1) {
         LOG.warn(DFSConfigKeys
             .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
-            " value cannot be less than 0. Disabling file IO profiling.");
-        isEnabled = false;
-      } else if (fileIOSamplingFraction > 1) {
-        LOG.warn(DFSConfigKeys
-            .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
-            " value cannot be more than 1. Setting value to 1 and enabling " +
-            "file IO profiling");
-        isEnabled = true;
+            " value cannot be more than 1. Setting value to 1");
         fileIOSamplingFraction = 1;
-      } else {
-        LOG.info(DFSConfigKeys
-            .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
-            + fileIOSamplingFraction + ". Enabling file IO profiling");
-        isEnabled = true;
       }
       sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
     } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d737a26c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 03e1dee..6a8ac9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
 
   private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
     Configuration conf = new HdfsConfiguration();
-    conf.setDouble(DFSConfigKeys
-        .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+    conf.setBoolean(DFSConfigKeys
+        .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
     SimulatedFSDataset.setFactory(conf);
     return new MiniDFSCluster.Builder(conf)
         .numDataNodes(NUM_DATANODES)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d737a26c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 6b6c87a..da5ef4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -107,6 +107,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase
{
         .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
     configurationPropsToSkipCompare
         .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
     configurationPropsToSkipCompare.add(DFSConfigKeys
         .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
     configurationPropsToSkipCompare.add(DFSConfigKeys


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message