hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jzh...@apache.org
Subject hadoop git commit: HADOOP-14844. Remove requirement to specify TenantGuid for MSI Token Provider. Contributed by Atul Sikaria.
Date Fri, 08 Sep 2017 20:04:38 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 720718eec -> bb107f5d2


HADOOP-14844. Remove requirement to specify TenantGuid for MSI Token Provider. Contributed
by Atul Sikaria.

(cherry picked from commit a4661850c1e0794baf493a468191e12681d68ab4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb107f5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb107f5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb107f5d

Branch: refs/heads/branch-3.0
Commit: bb107f5d28a98b78c9821c9c06e2e0b19b0ad54a
Parents: 720718e
Author: John Zhuge <jzhuge@apache.org>
Authored: Fri Sep 8 12:57:58 2017 -0700
Committer: John Zhuge <jzhuge@apache.org>
Committed: Fri Sep 8 12:57:58 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/core-default.xml         | 16 +++------------
 hadoop-tools/hadoop-azure-datalake/pom.xml      |  2 +-
 .../org/apache/hadoop/fs/adl/AdlConfKeys.java   |  1 -
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |  4 +---
 .../src/site/markdown/index.md                  | 21 ++++++--------------
 .../hadoop/fs/adl/TestAzureADTokenProvider.java |  4 ----
 6 files changed, 11 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb107f5d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 23739b0..269a13c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2717,8 +2717,7 @@
       fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
       The RefreshToken type requires property fs.adl.oauth2.client.id and
       fs.adl.oauth2.refresh.token.
-      The MSI type requires properties fs.adl.oauth2.msi.port and
-      fs.adl.oauth2.msi.tenantguid.
+      The MSI type reads optional property fs.adl.oauth2.msi.port, if specified.
       The DeviceCode type requires property
       fs.adl.oauth2.devicecode.clientapp.id.
       The Custom type requires property fs.adl.oauth2.access.token.provider.
@@ -2762,17 +2761,8 @@
     <value></value>
     <description>
       The localhost port for the MSI token service. This is the port specified
-      when creating the Azure VM.
-      Used by MSI token provider.
-    </description>
-  </property>
-
-  <property>
-    <name>fs.adl.oauth2.msi.tenantguid</name>
-    <value></value>
-    <description>
-      The tenant guid for the Azure AAD tenant under which the azure data lake
-      store account is created.
+      when creating the Azure VM. The default, if this setting is not specified,
+      is 50342.
       Used by MSI token provider.
     </description>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb107f5d/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index f699464..53b7b5e 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -110,7 +110,7 @@
     <dependency>
       <groupId>com.microsoft.azure</groupId>
       <artifactId>azure-data-lake-store-sdk</artifactId>
-      <version>2.2.2</version>
+      <version>2.2.3</version>
     </dependency>
     <!--  ENDS HERE-->
     <dependency>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb107f5d/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index f77d981..790902c 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -56,7 +56,6 @@ public final class AdlConfKeys {
 
   // MSI Auth Configuration
   public static final String MSI_PORT = "fs.adl.oauth2.msi.port";
-  public static final String MSI_TENANT_GUID = "fs.adl.oauth2.msi.tenantguid";
 
   // DeviceCode Auth configuration
   public static final String DEVICE_CODE_CLIENT_APP_ID =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb107f5d/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index a5e31e1..a496595 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -290,9 +290,7 @@ public class AdlFileSystem extends FileSystem {
 
   private AccessTokenProvider getMsiBasedTokenProvider(
           Configuration conf) throws IOException {
-    int port = Integer.parseInt(getNonEmptyVal(conf, MSI_PORT));
-    String tenantGuid = getPasswordString(conf, MSI_TENANT_GUID);
-    return new MsiTokenProvider(port, tenantGuid);
+    return new MsiTokenProvider(conf.getInt(MSI_PORT, -1));
   }
 
   private AccessTokenProvider getDeviceCodeTokenProvider(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb107f5d/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index e34da36..ca79321 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -164,15 +164,11 @@ Identity extension within the VM. The advantage of doing this is that
the
 credentials are managed by the extension, and do not have to be put into
 core-site.xml.
 
-To use MSI, the following two steps are needed:
-1. Modify the VM deployment template to specify the port number of the token
- service exposed to localhost by the identity extension in the VM.
-2. Get your Azure ActiveDirectory Tenant ID:
-   1. Go to [the portal](https://portal.azure.com)
-   2. Under services in left nav, look for Azure Active Directory and click on it.
-   3. Click on Properties
-   4. Note down the GUID shown under "Directory ID" - this is your AAD tenant ID
-
+To use MSI, modify the VM deployment template to use the identity extension. Note the
+port number you specified in the template: this is the port number for the REST endpoint
+of the token service exposed to localhost by the identity extension in the VM. The default
+recommended port number is 50342 - if the recommended port number is used, then the msi.port
+setting below can be omitted in the configuration.
 
 ##### Configure core-site.xml
 Add the following properties to your `core-site.xml`
@@ -185,12 +181,7 @@ Add the following properties to your `core-site.xml`
 
 <property>
   <name>fs.adl.oauth2.msi.port</name>
-  <value>PORT NUMBER FROM STEP 1 ABOVE</value>
-</property>
-
-<property>
-  <name>fs.adl.oauth2.msi.TenantGuid</name>
-  <value>AAD TENANT ID GUID FROM STEP 2 ABOVE</value>
+  <value>PORT NUMBER FROM ABOVE (if different from the default of 50342)</value>
 </property>
 ```
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb107f5d/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 929b33a..12c2e3f 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -43,8 +43,6 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
 import static org.apache.hadoop.fs.adl.AdlConfKeys.DEVICE_CODE_CLIENT_APP_ID;
-import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_PORT;
-import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_TENANT_GUID;
 import static org.apache.hadoop.fs.adl.TokenProviderType.*;
 import static org.junit.Assert.assertEquals;
 
@@ -107,8 +105,6 @@ public class TestAzureADTokenProvider {
           throws IOException, URISyntaxException {
     Configuration conf = new Configuration();
     conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, MSI);
-    conf.set(MSI_PORT, "54321");
-    conf.set(MSI_TENANT_GUID, "TENANT_GUID");
 
     URI uri = new URI("adl://localhost:8080");
     AdlFileSystem fileSystem = new AdlFileSystem();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message