hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject git commit: HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. Contributed by Shanyu Zhao.
Date Mon, 03 Nov 2014 22:32:59 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk ec6cbece8 -> 734eeb4f3


HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. Contributed
by Shanyu Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/734eeb4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/734eeb4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/734eeb4f

Branch: refs/heads/trunk
Commit: 734eeb4f357ad3210355a0d3fdbc80706a770d61
Parents: ec6cbec
Author: cnauroth <cnauroth@apache.org>
Authored: Mon Nov 3 14:29:18 2014 -0800
Committer: cnauroth <cnauroth@apache.org>
Committed: Mon Nov 3 14:29:18 2014 -0800

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../hadoop/fs/azure/NativeAzureFileSystem.java  | 20 +++++++++++++-------
 .../TestNativeAzureFileSystemMetricsSystem.java | 11 +++++++++++
 3 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 93e58f6..eb91dcb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -339,6 +339,9 @@ Trunk (Unreleased)
     HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., 
     sbin) (aw)
 
+    HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics
+    collection. (Shanyu Zhao via cnauroth)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 076c48a..ad2e2e6 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -641,6 +641,8 @@ public class NativeAzureFileSystem extends FileSystem {
   static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME =
       "fs.azure.output.stream.buffer.size";
 
+  public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics";
+
   private class NativeAzureFsInputStream extends FSInputStream {
     private InputStream in;
     private final String key;
@@ -1035,13 +1037,15 @@ public class NativeAzureFileSystem extends FileSystem {
       store = createDefaultStore(conf);
     }
 
-    // Make sure the metrics system is available before interacting with Azure
-    AzureFileSystemMetricsSystem.fileSystemStarted();
-    metricsSourceName = newMetricsSourceName();
-    String sourceDesc = "Azure Storage Volume File System metrics";
     instrumentation = new AzureFileSystemInstrumentation(conf);
-    AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
+    if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
+      // Make sure the metrics system is available before interacting with Azure
+      AzureFileSystemMetricsSystem.fileSystemStarted();
+      metricsSourceName = newMetricsSourceName();
+      String sourceDesc = "Azure Storage Volume File System metrics";
+      AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
         instrumentation);
+    }
 
     store.initialize(uri, conf, instrumentation);
     setConf(conf);
@@ -2207,8 +2211,10 @@ public class NativeAzureFileSystem extends FileSystem {
 
     long startTime = System.currentTimeMillis();
 
-    AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
-    AzureFileSystemMetricsSystem.fileSystemClosed();
+    if(!getConf().getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
+      AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName);
+      AzureFileSystemMetricsSystem.fileSystemClosed();
+    }
 
     if (LOG.isDebugEnabled()) {
         LOG.debug("Submitting metrics when file system closed took "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/734eeb4f/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
index f44613d..7820b7e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java
@@ -74,4 +74,15 @@ public class TestNativeAzureFileSystemMetricsSystem {
     assertTrue(name2.startsWith("AzureFileSystemMetrics"));
     assertTrue(!name1.equals(name2));
   }
+
+  @Test
+  public void testSkipMetricsCollection() throws Exception {
+    AzureBlobStorageTestAccount a;
+    a = AzureBlobStorageTestAccount.createMock();
+    a.getFileSystem().getConf().setBoolean(
+      NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, true);
+    a.getFileSystem().create(new Path("/foo")).close();
+    a.closeFileSystem(); // Causes the file system to close, which publishes metrics
+    assertEquals(0, getFilesCreated(a));
+  }
 }


Mime
View raw message