Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B1971200C57 for ; Sat, 15 Apr 2017 21:10:27 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B0445160BAC; Sat, 15 Apr 2017 19:10:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A71B3160BA9 for ; Sat, 15 Apr 2017 21:10:26 +0200 (CEST) Received: (qmail 5069 invoked by uid 500); 15 Apr 2017 19:10:23 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 4527 invoked by uid 99); 15 Apr 2017 19:10:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 15 Apr 2017 19:10:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 37664DFF4C; Sat, 15 Apr 2017 19:10:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: virajith@apache.org To: common-commits@hadoop.apache.org Date: Sat, 15 Apr 2017 19:10:27 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [06/50] [abbrv] hadoop git commit: HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge. archived-at: Sat, 15 Apr 2017 19:10:27 -0000 HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge. Signed-off-by: John Zhuge Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56e81f2a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56e81f2a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56e81f2a Branch: refs/heads/HDFS-9806 Commit: 56e81f2a204d5e65d29222df1b2ba4d892f9c0d5 Parents: 96cbb4f Author: John Zhuge Authored: Tue Mar 28 09:40:14 2017 -0700 Committer: John Zhuge Committed: Fri Apr 7 12:04:44 2017 -0700 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 63 ++++++++++++++++++++ .../conf/TestCommonConfigurationFields.java | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 2 + .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 3 +- .../apache/hadoop/fs/adl/AdlMockWebServer.java | 3 + .../hadoop/fs/adl/TestAzureADTokenProvider.java | 3 + .../hadoop/fs/adl/TestCustomTokenProvider.java | 4 ++ .../fs/adl/TestRelativePathFormation.java | 4 ++ 8 files changed, 82 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index b8f9904..521b013 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2456,6 +2456,7 @@ + fs.adl.impl org.apache.hadoop.fs.adl.AdlFileSystem @@ -2465,6 +2466,68 @@ fs.AbstractFileSystem.adl.impl org.apache.hadoop.fs.adl.Adl + + + adl.feature.ownerandgroup.enableupn + false + + When true : User and Group in FileStatus/AclStatus response is + represented as user friendly name as per Azure AD profile. + + When false (default) : User and Group in FileStatus/AclStatus + response is represented by the unique identifier from Azure AD + profile (Object ID as GUID). + + For optimal performance, false is recommended. + + + + + fs.adl.oauth2.access.token.provider.type + ClientCredential + + Defines Azure Active Directory OAuth2 access token provider type. + Supported types are ClientCredential, RefreshToken, and Custom. + The ClientCredential type requires property fs.adl.oauth2.client.id, + fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. + The RefreshToken type requires property fs.adl.oauth2.client.id and + fs.adl.oauth2.refresh.token. + The Custom type requires property fs.adl.oauth2.access.token.provider. + + + + + fs.adl.oauth2.client.id + + The OAuth2 client id. + + + + fs.adl.oauth2.credential + + The OAuth2 access key. + + + + fs.adl.oauth2.refresh.url + + The OAuth2 token endpoint. + + + + fs.adl.oauth2.refresh.token + + The OAuth2 refresh token. + + + + fs.adl.oauth2.access.token.provider + + + The class name of the OAuth2 access token provider. + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index cbfb6d1..8524973 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -105,7 +105,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { // ADL properties are in a different subtree // - org.apache.hadoop.hdfs.web.ADLConfKeys xmlPrefixToSkipCompare.add("adl."); - xmlPropsToSkipCompare.add("fs.adl.impl"); + xmlPrefixToSkipCompare.add("fs.adl."); xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl"); // Azure properties are in a different class http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index d3a5565..31df222 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -40,6 +40,8 @@ public final class AdlConfKeys { "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = "fs.adl.oauth2.access.token.provider.type"; + public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT = + TokenProviderType.ClientCredential; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 0b860b3..e63f115 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -243,7 +243,8 @@ public class AdlFileSystem extends FileSystem { Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( config, AdlFileSystem.class); TokenProviderType type = conf.getEnum( - AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT); switch (type) { case RefreshToken: http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java index 55c8f81..d843d55 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider; import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -84,6 +86,7 @@ public class AdlMockWebServer { // Responses are returned in the same order that they are enqueued. fs = new TestableAdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 3867e74..36498c6 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -101,6 +101,7 @@ public class TestAzureADTokenProvider { public void testCustomCredTokenProvider() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); @@ -115,6 +116,7 @@ public class TestAzureADTokenProvider { public void testInvalidProviderConfigurationForType() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); try { @@ -136,6 +138,7 @@ public class TestAzureADTokenProvider { Configuration conf = new Configuration(); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "wrong.classpath.CustomMockTokenProvider"); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java index c594c65..737534c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java @@ -38,6 +38,8 @@ import java.util.Collection; import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * Test access token provider behaviour with custom token provider and for token @@ -89,6 +91,8 @@ public class TestCustomTokenProvider extends AdlMockWebServer { */ public void init() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, typeOfTokenProviderClass.getName()); fileSystems = new TestableAdlFileSystem[fsObjectCount]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java index 908f8b8..334c372 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java @@ -29,6 +29,8 @@ import java.net.URISyntaxException; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * This class verifies path conversion to SDK. @@ -39,6 +41,8 @@ public class TestRelativePathFormation { public void testToRelativePath() throws URISyntaxException, IOException { AdlFileSystem fs = new AdlFileSystem(); Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org