Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F3436200BFF for ; Tue, 17 Jan 2017 07:52:17 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id F22F8160B46; Tue, 17 Jan 2017 06:52:17 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2241E160B43 for ; Tue, 17 Jan 2017 07:52:16 +0100 (CET) Received: (qmail 7772 invoked by uid 500); 17 Jan 2017 06:52:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 7763 invoked by uid 99); 17 Jan 2017 06:52:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Jan 2017 06:52:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 176A9DFADF; Tue, 17 Jan 2017 06:52:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Message-Id: <7986d131004241d1be4303d91addc3a0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-11339. Support File IO sampling for Datanode IO profiling hooks. Contributed by Hanisha Koneru. Date: Tue, 17 Jan 2017 06:52:16 +0000 (UTC) archived-at: Tue, 17 Jan 2017 06:52:18 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2 52fdd387e -> be8e001f1 HDFS-11339. Support File IO sampling for Datanode IO profiling hooks. Contributed by Hanisha Koneru. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be8e001f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be8e001f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be8e001f Branch: refs/heads/branch-2 Commit: be8e001f1f1cf9c84e2c213bd619374d38020e5c Parents: 52fdd38 Author: Arpit Agarwal Authored: Mon Jan 16 14:43:29 2017 -0800 Committer: Arpit Agarwal Committed: Mon Jan 16 22:49:14 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +++++++++++---- .../server/datanode/ProfilingFileIoEvents.java | 21 ++++++++++++++++++-- .../hadoop/tools/TestHdfsConfigFields.java | 2 ++ 3 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/be8e001f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index fb3acf7..d4ec567 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -642,10 +642,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY = "dfs.image.transfer.chunksize"; public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024; - //Keys with no defaults - public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; - public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory"; - public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy"; + // Datanode File IO Stats public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY = "dfs.datanode.enable.fileio.profiling"; public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT = @@ -654,6 +651,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.enable.fileio.fault.injection"; public static final boolean DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_DEFAULT = false; + public static final String + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY = + "dfs.datanode.fileio.profiling.sampling.fraction"; + public static final double + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0; + + //Keys with no defaults + public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; + public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory"; + public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy"; + public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold"; public static final long DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT = 1024L * 1024L * 1024L * 10L; // 10 GB public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/be8e001f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java index 43ac495..61d0432 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -26,6 +28,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.util.Time; import javax.annotation.Nullable; +import java.util.concurrent.ThreadLocalRandom; /** * Profiles the performance of the metadata and data related operations on @@ -33,16 +36,30 @@ import javax.annotation.Nullable; */ @InterfaceAudience.Private class ProfilingFileIoEvents { + static final Log LOG = LogFactory.getLog(ProfilingFileIoEvents.class); private final boolean isEnabled; + private final int sampleRangeMax; public ProfilingFileIoEvents(@Nullable Configuration conf) { if (conf != null) { isEnabled = conf.getBoolean(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); + double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT); + if (fileIOSamplingFraction > 1) { + LOG.warn(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + + " value cannot be more than 1. Setting value to 1"); + fileIOSamplingFraction = 1; + } + sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE); } else { isEnabled = false; + sampleRangeMax = 0; } } @@ -69,7 +86,7 @@ class ProfilingFileIoEvents { public long beforeFileIo(@Nullable FsVolumeSpi volume, FileIoProvider.OPERATION op, long len) { - if (isEnabled) { + if (isEnabled && ThreadLocalRandom.current().nextInt() < sampleRangeMax) { DataNodeVolumeMetrics metrics = getVolumeMetrics(volume); if (metrics != null) { return Time.monotonicNow(); @@ -80,7 +97,7 @@ class ProfilingFileIoEvents { public void afterFileIo(@Nullable FsVolumeSpi volume, FileIoProvider.OPERATION op, long begin, long len) { - if (isEnabled) { + if (isEnabled && begin != 0) { DataNodeVolumeMetrics metrics = getVolumeMetrics(volume); if (metrics != null) { long latency = Time.monotonicNow() - begin; http://git-wip-us.apache.org/repos/asf/hadoop/blob/be8e001f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java index 71ad713..da10878 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java @@ -107,6 +107,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase { .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY); + configurationPropsToSkipCompare.add(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY); // Allocate xmlPropsToSkipCompare = new HashSet(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org