Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8B6B317734 for ; Mon, 3 Nov 2014 22:33:00 +0000 (UTC) Received: (qmail 41340 invoked by uid 500); 3 Nov 2014 22:33:00 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 41253 invoked by uid 500); 3 Nov 2014 22:33:00 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 41232 invoked by uid 99); 3 Nov 2014 22:33:00 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Nov 2014 22:33:00 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E75CEA0627E; Mon, 3 Nov 2014 22:32:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cnauroth@apache.org To: common-commits@hadoop.apache.org Message-Id: <2e2496beb95e472892436bc5e12e822a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. Contributed by Shanyu Zhao. Date: Mon, 3 Nov 2014 22:32:59 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk ec6cbece8 -> 734eeb4f3 HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. Contributed by Shanyu Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/734eeb4f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/734eeb4f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/734eeb4f Branch: refs/heads/trunk Commit: 734eeb4f357ad3210355a0d3fdbc80706a770d61 Parents: ec6cbec Author: cnauroth Authored: Mon Nov 3 14:29:18 2014 -0800 Committer: cnauroth Committed: Mon Nov 3 14:29:18 2014 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop/fs/azure/NativeAzureFileSystem.java | 20 +++++++++++++------- .../TestNativeAzureFileSystemMetricsSystem.java | 11 +++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 93e58f6..eb91dcb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -339,6 +339,9 @@ Trunk (Unreleased) HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., sbin) (aw) + HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics + collection. (Shanyu Zhao via cnauroth) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 076c48a..ad2e2e6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -641,6 +641,8 @@ public class NativeAzureFileSystem extends FileSystem { static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME = "fs.azure.output.stream.buffer.size"; + public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics"; + private class NativeAzureFsInputStream extends FSInputStream { private InputStream in; private final String key; @@ -1035,13 +1037,15 @@ public class NativeAzureFileSystem extends FileSystem { store = createDefaultStore(conf); } - // Make sure the metrics system is available before interacting with Azure - AzureFileSystemMetricsSystem.fileSystemStarted(); - metricsSourceName = newMetricsSourceName(); - String sourceDesc = "Azure Storage Volume File System metrics"; instrumentation = new AzureFileSystemInstrumentation(conf); - AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, + if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { + // Make sure the metrics system is available before interacting with Azure + AzureFileSystemMetricsSystem.fileSystemStarted(); + metricsSourceName = newMetricsSourceName(); + String sourceDesc = "Azure Storage Volume File System metrics"; + AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, instrumentation); + } store.initialize(uri, conf, instrumentation); setConf(conf); @@ -2207,8 +2211,10 @@ public class NativeAzureFileSystem extends FileSystem { long startTime = System.currentTimeMillis(); - AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName); - AzureFileSystemMetricsSystem.fileSystemClosed(); + if(!getConf().getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { + AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName); + AzureFileSystemMetricsSystem.fileSystemClosed(); + } if (LOG.isDebugEnabled()) { LOG.debug("Submitting metrics when file system closed took " http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java index f44613d..7820b7e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java @@ -74,4 +74,15 @@ public class TestNativeAzureFileSystemMetricsSystem { assertTrue(name2.startsWith("AzureFileSystemMetrics")); assertTrue(!name1.equals(name2)); } + + @Test + public void testSkipMetricsCollection() throws Exception { + AzureBlobStorageTestAccount a; + a = AzureBlobStorageTestAccount.createMock(); + a.getFileSystem().getConf().setBoolean( + NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, true); + a.getFileSystem().create(new Path("/foo")).close(); + a.closeFileSystem(); // Causes the file system to close, which publishes metrics + assertEquals(0, getFilesCreated(a)); + } }