Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B47F9200C4C for ; Tue, 4 Apr 2017 21:56:33 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B322E160BA2; Tue, 4 Apr 2017 19:56:33 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 5F068160BA1 for ; Tue, 4 Apr 2017 21:56:32 +0200 (CEST) Received: (qmail 97371 invoked by uid 500); 4 Apr 2017 19:56:25 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 97126 invoked by uid 99); 4 Apr 2017 19:56:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Apr 2017 19:56:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9E8E2E00C4; Tue, 4 Apr 2017 19:56:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Tue, 04 Apr 2017 19:56:28 -0000 Message-Id: In-Reply-To: <83db38e6376c4b2a8806af407cf6bf57@git.apache.org> References: <83db38e6376c4b2a8806af407cf6bf57@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/50] [abbrv] hadoop git commit: HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge. archived-at: Tue, 04 Apr 2017 19:56:33 -0000 HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/253e3e78 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/253e3e78 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/253e3e78 Branch: refs/heads/HDFS-7240 Commit: 253e3e78abb9ebe0e8103d25538ff07875b33c84 Parents: 448ec81 Author: John Zhuge Authored: Mon Feb 20 21:27:46 2017 -0800 Committer: John Zhuge Committed: Mon Mar 27 23:05:57 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 15 +++- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 38 +++++++-- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 + .../src/site/markdown/index.md | 22 +++--- .../hadoop/fs/adl/TestAzureADTokenProvider.java | 5 +- .../fs/adl/TestValidateConfiguration.java | 83 ++++++++++++++++++-- 6 files changed, 138 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index ada8b02..da4f6ac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -751,7 +751,20 @@ public class Configuration implements Iterable>, this.loadDefaults = other.loadDefaults; setQuietMode(other.getQuietMode()); } - + + /** + * Reload existing configuration instances. + */ + public static synchronized void reloadExistingConfigurations() { + if (LOG.isDebugEnabled()) { + LOG.debug("Reloading " + REGISTRY.keySet().size() + + " existing configurations"); + } + for (Configuration conf : REGISTRY.keySet()) { + conf.reloadConfiguration(); + } + } + /** * Add a default resource. Resources are loaded in the order of the resources * added. http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index 8fc8e00..d3a5565 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configuration.DeprecationDelta; /** * Constants. @@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceStability.Evolving public final class AdlConfKeys { // OAuth2 Common Configuration - public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2" - + ".refresh.url"; + public static final String AZURE_AD_REFRESH_URL_KEY = + "fs.adl.oauth2.refresh.url"; // optional when provider type is refresh or client id. public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY = - "dfs.adls.oauth2.access.token.provider"; + "fs.adl.oauth2.access.token.provider"; public static final String AZURE_AD_CLIENT_ID_KEY = - "dfs.adls.oauth2.client.id"; + "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = - "dfs.adls.oauth2.access.token.provider.type"; + "fs.adl.oauth2.access.token.provider.type"; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = - "dfs.adls.oauth2.refresh.token"; + "fs.adl.oauth2.refresh.token"; public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken"; // OAuth Client Cred Token Configuration public static final String AZURE_AD_CLIENT_SECRET_KEY = - "dfs.adls.oauth2.credential"; + "fs.adl.oauth2.credential"; public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED = "ClientCredential"; @@ -75,7 +77,7 @@ public final class AdlConfKeys { static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024; static final String LATENCY_TRACKER_KEY = - "adl.dfs.enable.client.latency.tracker"; + "adl.enable.client.latency.tracker"; static final boolean LATENCY_TRACKER_DEFAULT = true; static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY = @@ -90,6 +92,26 @@ public final class AdlConfKeys { "adl.feature.ownerandgroup.enableupn"; static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false; + public static void addDeprecatedKeys() { + Configuration.addDeprecations(new DeprecationDelta[]{ + new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type", + AZURE_AD_TOKEN_PROVIDER_TYPE_KEY), + new DeprecationDelta("dfs.adls.oauth2.client.id", + AZURE_AD_CLIENT_ID_KEY), + new DeprecationDelta("dfs.adls.oauth2.refresh.token", + AZURE_AD_REFRESH_TOKEN_KEY), + new DeprecationDelta("dfs.adls.oauth2.refresh.url", + AZURE_AD_REFRESH_URL_KEY), + new DeprecationDelta("dfs.adls.oauth2.credential", + AZURE_AD_CLIENT_SECRET_KEY), + new DeprecationDelta("dfs.adls.oauth2.access.token.provider", + AZURE_AD_TOKEN_PROVIDER_CLASS_KEY), + new DeprecationDelta("adl.dfs.enable.client.latency.tracker", + LATENCY_TRACKER_KEY) + }); + Configuration.reloadExistingConfigurations(); + } + private AdlConfKeys() { } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index e0e273e..0b860b3 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem { private AccessTokenProvider tokenProvider; private AzureADTokenProvider azureTokenProvider; + static { + AdlConfKeys.addDeprecatedKeys(); + } + @Override public String getScheme() { return SCHEME; http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index de8fda2..d2da858 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml` ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type RefreshToken ``` @@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*] ```xml - dfs.adls.oauth2.client.id + fs.adl.oauth2.client.id - dfs.adls.oauth2.refresh.token + fs.adl.oauth2.refresh.token ``` @@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml` ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type ClientCredential - dfs.adls.oauth2.refresh.url + fs.adl.oauth2.refresh.url TOKEN ENDPOINT FROM STEP 7 ABOVE - dfs.adls.oauth2.client.id + fs.adl.oauth2.client.id CLIENT ID FROM STEP 7 ABOVE - dfs.adls.oauth2.credential + fs.adl.oauth2.credential PASSWORD FROM STEP 7 ABOVE ``` @@ -166,9 +166,9 @@ For additional reading on the credential provider API, see ##### Provisioning ```bash -hadoop credential create dfs.adls.oauth2.client.id -value 123 +hadoop credential create fs.adl.oauth2.client.id -value 123 -provider localjceks://file/home/foo/adls.jceks -hadoop credential create dfs.adls.oauth2.refresh.token -value 123 +hadoop credential create fs.adl.oauth2.refresh.token -value 123 -provider localjceks://file/home/foo/adls.jceks ``` @@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123 ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type RefreshToken @@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123 ```bash hadoop distcp - [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken + [-D fs.adl.oauth2.access.token.provider.type=RefreshToken -D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks] hdfs://:9001/user/foo/srcDir adl://.azuredatalakestore.net/tgtDir/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 70f2a7f..3867e74 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -121,8 +122,8 @@ public class TestAzureADTokenProvider { Assert.fail("Initialization should have failed due no token provider " + "configuration"); } catch (IllegalArgumentException e) { - Assert.assertTrue( - e.getMessage().contains("dfs.adls.oauth2.access.token.provider")); + GenericTestUtils.assertExceptionContains( + AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e); } conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/253e3e78/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java index 4cabaa3..3d51b42 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.adl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; @@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys .TOKEN_PROVIDER_TYPE_REFRESH_TOKEN; import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + /** * Validate configuration keys defined for adl storage file system instance. */ @@ -64,18 +71,18 @@ public class TestValidateConfiguration { @Test public void validateConfigurationKeys() { Assert - .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY); - Assert.assertEquals("dfs.adls.oauth2.access.token.provider", + .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY); + Assert.assertEquals("fs.adl.oauth2.access.token.provider", AZURE_AD_TOKEN_PROVIDER_CLASS_KEY); - Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY); - Assert.assertEquals("dfs.adls.oauth2.refresh.token", + Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY); + Assert.assertEquals("fs.adl.oauth2.refresh.token", AZURE_AD_REFRESH_TOKEN_KEY); Assert - .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY); + .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY); Assert.assertEquals("adl.debug.override.localuserasfileowner", ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER); - Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type", + Assert.assertEquals("fs.adl.oauth2.access.token.provider.type", AZURE_AD_TOKEN_PROVIDER_TYPE_KEY); Assert.assertEquals("adl.feature.client.cache.readahead", @@ -88,7 +95,7 @@ public class TestValidateConfiguration { Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED); - Assert.assertEquals("adl.dfs.enable.client.latency.tracker", + Assert.assertEquals("adl.enable.client.latency.tracker", LATENCY_TRACKER_KEY); Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT); @@ -109,4 +116,66 @@ public class TestValidateConfiguration { Assert.assertEquals(false, ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT); } + + @Test + public void testSetDeprecatedKeys() throws ClassNotFoundException { + Configuration conf = new Configuration(true); + setDeprecatedKeys(conf); + + // Force AdlFileSystem static initialization to register deprecated keys. + Class.forName(AdlFileSystem.class.getName()); + + assertDeprecatedKeys(conf); + } + + @Test + public void testLoadDeprecatedKeys() + throws IOException, ClassNotFoundException { + Configuration saveConf = new Configuration(false); + setDeprecatedKeys(saveConf); + + final File testRootDir = GenericTestUtils.getTestDir(); + File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml"); + OutputStream out = new FileOutputStream(confXml); + saveConf.writeXml(out); + out.close(); + + Configuration conf = new Configuration(true); + conf.addResource(confXml.toURI().toURL()); + + // Trigger loading the configuration resources by getting any key. + conf.get("dummy.key"); + + // Force AdlFileSystem static initialization to register deprecated keys. + Class.forName(AdlFileSystem.class.getName()); + + assertDeprecatedKeys(conf); + } + + private void setDeprecatedKeys(Configuration conf) { + conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType"); + conf.set("dfs.adls.oauth2.client.id", "dummyClientId"); + conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken"); + conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl"); + conf.set("dfs.adls.oauth2.credential", "dummyCredential"); + conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass"); + conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker"); + } + + private void assertDeprecatedKeys(Configuration conf) { + Assert.assertEquals("dummyType", + conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY)); + Assert.assertEquals("dummyClientId", + conf.get(AZURE_AD_CLIENT_ID_KEY)); + Assert.assertEquals("dummyRefreshToken", + conf.get(AZURE_AD_REFRESH_TOKEN_KEY)); + Assert.assertEquals("dummyRefreshUrl", + conf.get(AZURE_AD_REFRESH_URL_KEY)); + Assert.assertEquals("dummyCredential", + conf.get(AZURE_AD_CLIENT_SECRET_KEY)); + Assert.assertEquals("dummyClass", + conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY)); + Assert.assertEquals("dummyTracker", + conf.get(LATENCY_TRACKER_KEY)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org