hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [12/50] hadoop git commit: HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge.
Date Wed, 12 Apr 2017 23:41:59 GMT
HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed
by John Zhuge.

Signed-off-by: John Zhuge <jzhuge@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56e81f2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56e81f2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56e81f2a

Branch: refs/heads/HDFS-7240
Commit: 56e81f2a204d5e65d29222df1b2ba4d892f9c0d5
Parents: 96cbb4f
Author: John Zhuge <jzhuge@apache.org>
Authored: Tue Mar 28 09:40:14 2017 -0700
Committer: John Zhuge <jzhuge@apache.org>
Committed: Fri Apr 7 12:04:44 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/core-default.xml         | 63 ++++++++++++++++++++
 .../conf/TestCommonConfigurationFields.java     |  2 +-
 .../org/apache/hadoop/fs/adl/AdlConfKeys.java   |  2 +
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |  3 +-
 .../apache/hadoop/fs/adl/AdlMockWebServer.java  |  3 +
 .../hadoop/fs/adl/TestAzureADTokenProvider.java |  3 +
 .../hadoop/fs/adl/TestCustomTokenProvider.java  |  4 ++
 .../fs/adl/TestRelativePathFormation.java       |  4 ++
 8 files changed, 82 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b8f9904..521b013 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2456,6 +2456,7 @@
 
 
   <!-- Azure Data Lake File System Configurations -->
+
   <property>
     <name>fs.adl.impl</name>
     <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
@@ -2465,6 +2466,68 @@
     <name>fs.AbstractFileSystem.adl.impl</name>
     <value>org.apache.hadoop.fs.adl.Adl</value>
   </property>
+
+  <property>
+    <name>adl.feature.ownerandgroup.enableupn</name>
+    <value>false</value>
+    <description>
+      When true : User and Group in FileStatus/AclStatus response is
+      represented as user friendly name as per Azure AD profile.
+
+      When false (default) : User and Group in FileStatus/AclStatus
+      response is represented by the unique identifier from Azure AD
+      profile (Object ID as GUID).
+
+      For optimal performance, false is recommended.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.adl.oauth2.access.token.provider.type</name>
+    <value>ClientCredential</value>
+    <description>
+      Defines Azure Active Directory OAuth2 access token provider type.
+      Supported types are ClientCredential, RefreshToken, and Custom.
+      The ClientCredential type requires property fs.adl.oauth2.client.id,
+      fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
+      The RefreshToken type requires property fs.adl.oauth2.client.id and
+      fs.adl.oauth2.refresh.token.
+      The Custom type requires property fs.adl.oauth2.access.token.provider.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.adl.oauth2.client.id</name>
+    <value></value>
+    <description>The OAuth2 client id.</description>
+  </property>
+
+  <property>
+    <name>fs.adl.oauth2.credential</name>
+    <value></value>
+    <description>The OAuth2 access key.</description>
+  </property>
+
+  <property>
+    <name>fs.adl.oauth2.refresh.url</name>
+    <value></value>
+    <description>The OAuth2 token endpoint.</description>
+  </property>
+
+  <property>
+    <name>fs.adl.oauth2.refresh.token</name>
+    <value></value>
+    <description>The OAuth2 refresh token.</description>
+  </property>
+
+  <property>
+    <name>fs.adl.oauth2.access.token.provider</name>
+    <value></value>
+    <description>
+      The class name of the OAuth2 access token provider.
+    </description>
+  </property>
+
   <!-- Azure Data Lake File System Configurations Ends Here-->
 
   <property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index cbfb6d1..8524973 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -105,7 +105,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase
{
     // ADL properties are in a different subtree
     // - org.apache.hadoop.hdfs.web.ADLConfKeys
     xmlPrefixToSkipCompare.add("adl.");
-    xmlPropsToSkipCompare.add("fs.adl.impl");
+    xmlPrefixToSkipCompare.add("fs.adl.");
     xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
 
     // Azure properties are in a different class

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index d3a5565..31df222 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -40,6 +40,8 @@ public final class AdlConfKeys {
       "fs.adl.oauth2.client.id";
   public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
       "fs.adl.oauth2.access.token.provider.type";
+  public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT =
+      TokenProviderType.ClientCredential;
 
   // OAuth Refresh Token Configuration
   public static final String AZURE_AD_REFRESH_TOKEN_KEY =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 0b860b3..e63f115 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -243,7 +243,8 @@ public class AdlFileSystem extends FileSystem {
     Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
         config, AdlFileSystem.class);
     TokenProviderType type = conf.getEnum(
-        AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
+        AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
+        AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT);
 
     switch (type) {
     case RefreshToken:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
index 55c8f81..d843d55 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
 import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+    .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
 
 import com.squareup.okhttp.mockwebserver.MockWebServer;
 
@@ -84,6 +86,7 @@ public class AdlMockWebServer {
     // Responses are returned in the same order that they are enqueued.
     fs = new TestableAdlFileSystem();
 
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
     conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         CustomMockTokenProvider.class, AzureADTokenProvider.class);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 3867e74..36498c6 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -101,6 +101,7 @@ public class TestAzureADTokenProvider {
   public void testCustomCredTokenProvider()
       throws URISyntaxException, IOException {
     Configuration conf = new Configuration();
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
     conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         CustomMockTokenProvider.class, AzureADTokenProvider.class);
 
@@ -115,6 +116,7 @@ public class TestAzureADTokenProvider {
   public void testInvalidProviderConfigurationForType()
       throws URISyntaxException, IOException {
     Configuration conf = new Configuration();
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
     URI uri = new URI("adl://localhost:8080");
     AdlFileSystem fileSystem = new AdlFileSystem();
     try {
@@ -136,6 +138,7 @@ public class TestAzureADTokenProvider {
     Configuration conf = new Configuration();
     URI uri = new URI("adl://localhost:8080");
     AdlFileSystem fileSystem = new AdlFileSystem();
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
     conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         "wrong.classpath.CustomMockTokenProvider");
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
index c594c65..737534c 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
@@ -38,6 +38,8 @@ import java.util.Collection;
 import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+    .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
 
 /**
  * Test access token provider behaviour with custom token provider and for token
@@ -89,6 +91,8 @@ public class TestCustomTokenProvider extends AdlMockWebServer {
    */
   public void init() throws IOException, URISyntaxException {
     Configuration configuration = new Configuration();
+    configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
+        TokenProviderType.Custom);
     configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         typeOfTokenProviderClass.getName());
     fileSystems = new TestableAdlFileSystem[fsObjectCount];

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e81f2a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
index 908f8b8..334c372 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
@@ -29,6 +29,8 @@ import java.net.URISyntaxException;
 
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+    .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
 
 /**
  * This class verifies path conversion to SDK.
@@ -39,6 +41,8 @@ public class TestRelativePathFormation {
   public void testToRelativePath() throws URISyntaxException, IOException {
     AdlFileSystem fs = new AdlFileSystem();
     Configuration configuration = new Configuration();
+    configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
+        TokenProviderType.Custom);
     configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message