hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [04/50] [abbrv] hadoop git commit: HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge.
Date Tue, 04 Apr 2017 19:56:28 GMT
HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/253e3e78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/253e3e78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/253e3e78

Branch: refs/heads/HDFS-7240
Commit: 253e3e78abb9ebe0e8103d25538ff07875b33c84
Parents: 448ec81
Author: John Zhuge <jzhuge@cloudera.com>
Authored: Mon Feb 20 21:27:46 2017 -0800
Committer: John Zhuge <jzhuge@apache.org>
Committed: Mon Mar 27 23:05:57 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/conf/Configuration.java   | 15 +++-
 .../org/apache/hadoop/fs/adl/AdlConfKeys.java   | 38 +++++++--
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |  4 +
 .../src/site/markdown/index.md                  | 22 +++---
 .../hadoop/fs/adl/TestAzureADTokenProvider.java |  5 +-
 .../fs/adl/TestValidateConfiguration.java       | 83 ++++++++++++++++++--
 6 files changed, 138 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index ada8b02..da4f6ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -751,7 +751,20 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     this.loadDefaults = other.loadDefaults;
     setQuietMode(other.getQuietMode());
   }
-  
+
+  /**
+   * Reload existing configuration instances.
+   */
+  public static synchronized void reloadExistingConfigurations() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Reloading " + REGISTRY.keySet().size()
+          + " existing configurations");
+    }
+    for (Configuration conf : REGISTRY.keySet()) {
+      conf.reloadConfiguration();
+    }
+  }
+
   /**
    * Add a default resource. Resources are loaded in the order of the resources 
    * added.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 8fc8e00..d3a5565 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
 
 /**
  * Constants.
@@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceStability.Evolving
 public final class AdlConfKeys {
   // OAuth2 Common Configuration
-  public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2"
-      + ".refresh.url";
+  public static final String AZURE_AD_REFRESH_URL_KEY =
+      "fs.adl.oauth2.refresh.url";
 
   // optional when provider type is refresh or client id.
   public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY =
-      "dfs.adls.oauth2.access.token.provider";
+      "fs.adl.oauth2.access.token.provider";
   public static final String AZURE_AD_CLIENT_ID_KEY =
-      "dfs.adls.oauth2.client.id";
+      "fs.adl.oauth2.client.id";
   public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
-      "dfs.adls.oauth2.access.token.provider.type";
+      "fs.adl.oauth2.access.token.provider.type";
 
   // OAuth Refresh Token Configuration
   public static final String AZURE_AD_REFRESH_TOKEN_KEY =
-      "dfs.adls.oauth2.refresh.token";
+      "fs.adl.oauth2.refresh.token";
 
   public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken";
   // OAuth Client Cred Token Configuration
   public static final String AZURE_AD_CLIENT_SECRET_KEY =
-      "dfs.adls.oauth2.credential";
+      "fs.adl.oauth2.credential";
   public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
       "ClientCredential";
 
@@ -75,7 +77,7 @@ public final class AdlConfKeys {
   static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
 
   static final String LATENCY_TRACKER_KEY =
-      "adl.dfs.enable.client.latency.tracker";
+      "adl.enable.client.latency.tracker";
   static final boolean LATENCY_TRACKER_DEFAULT = true;
 
   static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY =
@@ -90,6 +92,26 @@ public final class AdlConfKeys {
       "adl.feature.ownerandgroup.enableupn";
   static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
 
+  public static void addDeprecatedKeys() {
+    Configuration.addDeprecations(new DeprecationDelta[]{
+        new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type",
+            AZURE_AD_TOKEN_PROVIDER_TYPE_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.client.id",
+            AZURE_AD_CLIENT_ID_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.refresh.token",
+            AZURE_AD_REFRESH_TOKEN_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.refresh.url",
+            AZURE_AD_REFRESH_URL_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.credential",
+            AZURE_AD_CLIENT_SECRET_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.access.token.provider",
+            AZURE_AD_TOKEN_PROVIDER_CLASS_KEY),
+        new DeprecationDelta("adl.dfs.enable.client.latency.tracker",
+            LATENCY_TRACKER_KEY)
+    });
+    Configuration.reloadExistingConfigurations();
+  }
+
   private AdlConfKeys() {
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index e0e273e..0b860b3 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem {
   private AccessTokenProvider tokenProvider;
   private AzureADTokenProvider azureTokenProvider;
 
+  static {
+    AdlConfKeys.addDeprecatedKeys();
+  }
+
   @Override
   public String getScheme() {
     return SCHEME;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index de8fda2..d2da858 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml`
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.access.token.provider.type</name>
+  <name>fs.adl.oauth2.access.token.provider.type</name>
   <value>RefreshToken</value>
 </property>
 ```
@@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library
For Java*]
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.client.id</name>
+  <name>fs.adl.oauth2.client.id</name>
   <value></value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.refresh.token</name>
+  <name>fs.adl.oauth2.refresh.token</name>
   <value></value>
 </property>
 ```
@@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml`
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.access.token.provider.type</name>
+  <name>fs.adl.oauth2.access.token.provider.type</name>
   <value>ClientCredential</value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.refresh.url</name>
+  <name>fs.adl.oauth2.refresh.url</name>
   <value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.client.id</name>
+  <name>fs.adl.oauth2.client.id</name>
   <value>CLIENT ID FROM STEP 7 ABOVE</value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.credential</name>
+  <name>fs.adl.oauth2.credential</name>
   <value>PASSWORD FROM STEP 7 ABOVE</value>
 </property>
 ```
@@ -166,9 +166,9 @@ For additional reading on the credential provider API, see
 ##### Provisioning
 
 ```bash
-hadoop credential create dfs.adls.oauth2.client.id -value 123
+hadoop credential create fs.adl.oauth2.client.id -value 123
     -provider localjceks://file/home/foo/adls.jceks
-hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+hadoop credential create fs.adl.oauth2.refresh.token -value 123
     -provider localjceks://file/home/foo/adls.jceks
 ```
 
@@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.access.token.provider.type</name>
+  <name>fs.adl.oauth2.access.token.provider.type</name>
   <value>RefreshToken</value>
 </property>
 <property>
@@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
 
 ```bash
 hadoop distcp
-    [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken
+    [-D fs.adl.oauth2.access.token.provider.type=RefreshToken
      -D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
     hdfs://<NameNode Hostname>:9001/user/foo/srcDir
     adl://<Account Name>.azuredatalakestore.net/tgtDir/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 70f2a7f..3867e74 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals;
 import org.apache.hadoop.security.ProviderUtils;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -121,8 +122,8 @@ public class TestAzureADTokenProvider {
       Assert.fail("Initialization should have failed due no token provider "
           + "configuration");
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(
-          e.getMessage().contains("dfs.adls.oauth2.access.token.provider"));
+      GenericTestUtils.assertExceptionContains(
+          AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e);
     }
     conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         CustomMockTokenProvider.class, AzureADTokenProvider.class);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
index 4cabaa3..3d51b42 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.adl;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
     .TOKEN_PROVIDER_TYPE_REFRESH_TOKEN;
 import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
 
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
 /**
  * Validate configuration keys defined for adl storage file system instance.
  */
@@ -64,18 +71,18 @@ public class TestValidateConfiguration {
   @Test
   public void validateConfigurationKeys() {
     Assert
-        .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
-    Assert.assertEquals("dfs.adls.oauth2.access.token.provider",
+        .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
+    Assert.assertEquals("fs.adl.oauth2.access.token.provider",
         AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
-    Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
-    Assert.assertEquals("dfs.adls.oauth2.refresh.token",
+    Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
+    Assert.assertEquals("fs.adl.oauth2.refresh.token",
         AZURE_AD_REFRESH_TOKEN_KEY);
     Assert
-        .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
+        .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
     Assert.assertEquals("adl.debug.override.localuserasfileowner",
         ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER);
 
-    Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type",
+    Assert.assertEquals("fs.adl.oauth2.access.token.provider.type",
         AZURE_AD_TOKEN_PROVIDER_TYPE_KEY);
 
     Assert.assertEquals("adl.feature.client.cache.readahead",
@@ -88,7 +95,7 @@ public class TestValidateConfiguration {
 
     Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED);
 
-    Assert.assertEquals("adl.dfs.enable.client.latency.tracker",
+    Assert.assertEquals("adl.enable.client.latency.tracker",
         LATENCY_TRACKER_KEY);
 
     Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT);
@@ -109,4 +116,66 @@ public class TestValidateConfiguration {
     Assert.assertEquals(false,
         ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
   }
+
+  @Test
+  public void testSetDeprecatedKeys() throws ClassNotFoundException {
+    Configuration conf = new Configuration(true);
+    setDeprecatedKeys(conf);
+
+    // Force AdlFileSystem static initialization to register deprecated keys.
+    Class.forName(AdlFileSystem.class.getName());
+
+    assertDeprecatedKeys(conf);
+  }
+
+  @Test
+  public void testLoadDeprecatedKeys()
+      throws IOException, ClassNotFoundException {
+    Configuration saveConf = new Configuration(false);
+    setDeprecatedKeys(saveConf);
+
+    final File testRootDir = GenericTestUtils.getTestDir();
+    File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml");
+    OutputStream out = new FileOutputStream(confXml);
+    saveConf.writeXml(out);
+    out.close();
+
+    Configuration conf = new Configuration(true);
+    conf.addResource(confXml.toURI().toURL());
+
+    // Trigger loading the configuration resources by getting any key.
+    conf.get("dummy.key");
+
+    // Force AdlFileSystem static initialization to register deprecated keys.
+    Class.forName(AdlFileSystem.class.getName());
+
+    assertDeprecatedKeys(conf);
+  }
+
+  private void setDeprecatedKeys(Configuration conf) {
+    conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType");
+    conf.set("dfs.adls.oauth2.client.id", "dummyClientId");
+    conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken");
+    conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl");
+    conf.set("dfs.adls.oauth2.credential", "dummyCredential");
+    conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass");
+    conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker");
+  }
+
+  private void assertDeprecatedKeys(Configuration conf) {
+    Assert.assertEquals("dummyType",
+        conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY));
+    Assert.assertEquals("dummyClientId",
+        conf.get(AZURE_AD_CLIENT_ID_KEY));
+    Assert.assertEquals("dummyRefreshToken",
+        conf.get(AZURE_AD_REFRESH_TOKEN_KEY));
+    Assert.assertEquals("dummyRefreshUrl",
+        conf.get(AZURE_AD_REFRESH_URL_KEY));
+    Assert.assertEquals("dummyCredential",
+        conf.get(AZURE_AD_CLIENT_SECRET_KEY));
+    Assert.assertEquals("dummyClass",
+        conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY));
+    Assert.assertEquals("dummyTracker",
+        conf.get(LATENCY_TRACKER_KEY));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message