hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject [08/10] hadoop git commit: HADOOP-13956. Read ADLS credentials from Credential Provider. (John Zhuge via lei)
Date Thu, 09 Mar 2017 23:20:08 GMT
HADOOP-13956. Read ADLS credentials from Credential Provider. (John Zhuge via lei)

(cherry picked from commit e015b563197a475e354bf84fd27e7bbcc67e00a4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01624f0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01624f0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01624f0c

Branch: refs/heads/branch-2
Commit: 01624f0c1b7eba9fe4b5778e1da9d38545af3722
Parents: 881b634
Author: Lei Xu <lei@apache.org>
Authored: Fri Jan 20 14:34:02 2017 +0800
Committer: Chris Douglas <cdouglas@apache.org>
Committed: Thu Mar 9 15:16:28 2017 -0800

----------------------------------------------------------------------
 .../src/site/markdown/CredentialProviderAPI.md  |   1 +
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |  37 ++++-
 .../src/site/markdown/index.md                  |  44 ++++++
 .../hadoop/fs/adl/TestAzureADTokenProvider.java | 152 +++++++++++++++++++
 4 files changed, 226 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/01624f0c/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
index a40bf2b..30dfdd8 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -102,6 +102,7 @@ In summary, first, provision the credentials into a provider then configure
the
 |YARN                 |WebAppUtils uptakes the use of the credential provider API through
the new method on Configuration called getPassword. This provides an alternative to storing
the passwords in clear text within the ssl-server.xml file while maintaining backward compatibility.|TODO|
 |AWS <br/> S3/S3A     |Uses Configuration.getPassword to get the S3 credentials. They
may be resolved through the credential provider API or from the config for backward compatibility.|[AWS
S3/S3A Usage](../../hadoop-aws/tools/hadoop-aws/index.html)|
 |Azure <br/> WASB     |Uses Configuration.getPassword to get the WASB credentials.
They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure
WASB Usage](../../hadoop-azure/index.html)|
+|Azure <br/> ADLS     |Uses Configuration.getPassword to get the ADLS credentials.
They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure
ADLS Usage](../../hadoop-azure-datalake/index.html)|
 |Apache <br/> Accumulo|The trace.password property is used by the Tracer to authenticate
with Accumulo and persist the traces in the trace table. The credential provider API is used
to acquire the trace.password from a provider or from configuration for backward compatibility.|TODO|
 |Apache <br/> Slider  |A capability has been added to Slider to prompt the user for
needed passwords and store them using CredentialProvider so they can be retrieved by an app
later.|TODO|
 |Apache <br/> Hive    |Protection of the metastore password, SSL related passwords
and JDO string password has been added through the use of the Credential Provider API|TODO|

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01624f0c/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index bd43c52..3d41025 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -58,10 +58,12 @@ import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ProviderUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.VersionInfo;
+
 import static org.apache.hadoop.fs.adl.AdlConfKeys.*;
 
 /**
@@ -224,8 +226,10 @@ public class AdlFileSystem extends FileSystem {
     return azureTokenProvider;
   }
 
-  private AccessTokenProvider getAccessTokenProvider(Configuration conf)
+  private AccessTokenProvider getAccessTokenProvider(Configuration config)
       throws IOException {
+    Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
+        config, AdlFileSystem.class);
     TokenProviderType type = conf.getEnum(
         AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
 
@@ -248,17 +252,17 @@ public class AdlFileSystem extends FileSystem {
   }
 
   private AccessTokenProvider getConfCredentialBasedTokenProvider(
-      Configuration conf) {
-    String clientId = getNonEmptyVal(conf, AZURE_AD_CLIENT_ID_KEY);
-    String refreshUrl = getNonEmptyVal(conf, AZURE_AD_REFRESH_URL_KEY);
-    String clientSecret = getNonEmptyVal(conf, AZURE_AD_CLIENT_SECRET_KEY);
+      Configuration conf) throws IOException {
+    String clientId = getPasswordString(conf, AZURE_AD_CLIENT_ID_KEY);
+    String refreshUrl = getPasswordString(conf, AZURE_AD_REFRESH_URL_KEY);
+    String clientSecret = getPasswordString(conf, AZURE_AD_CLIENT_SECRET_KEY);
     return new ClientCredsTokenProvider(refreshUrl, clientId, clientSecret);
   }
 
   private AccessTokenProvider getConfRefreshTokenBasedTokenProvider(
-      Configuration conf) {
-    String clientId = getNonEmptyVal(conf, AZURE_AD_CLIENT_ID_KEY);
-    String refreshToken = getNonEmptyVal(conf, AZURE_AD_REFRESH_TOKEN_KEY);
+      Configuration conf) throws IOException {
+    String clientId = getPasswordString(conf, AZURE_AD_CLIENT_ID_KEY);
+    String refreshToken = getPasswordString(conf, AZURE_AD_REFRESH_TOKEN_KEY);
     return new RefreshTokenBasedTokenProvider(clientId, refreshToken);
   }
 
@@ -938,4 +942,21 @@ public class AdlFileSystem extends FileSystem {
     return value;
   }
 
+  /**
+   * A wrapper of {@link Configuration#getPassword(String)}. It returns
+   * <code>String</code> instead of <code>char[]</code>.
+   *
+   * @param conf the configuration
+   * @param key the property key
+   * @return the password string
+   * @throws IOException if the password was not found
+   */
+  private static String getPasswordString(Configuration conf, String key)
+      throws IOException {
+    char[] passchars = conf.getPassword(key);
+    if (passchars == null) {
+      throw new IOException("Password " + key + " not found");
+    }
+    return new String(passchars);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01624f0c/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index 00825d1..ced5cff 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -23,6 +23,7 @@
     * [Configuring Credentials & FileSystem](#Configuring_Credentials)
         * [Using Refresh Token](#Refresh_Token)
         * [Using Client Keys](#Client_Credential_Token)
+        * [Protecting the Credentials with Credential Providers](#Credential_Provider)
     * [Enabling ADL Filesystem](#Enabling_ADL)
     * [Accessing adl URLs](#Accessing_adl_URLs)
 * [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
@@ -139,6 +140,49 @@ Add the following properties to your core-site.xml
     </property>
 
 
+### <a name="Credential_Provider" />Protecting the Credentials with Credential Providers
+
+In many Hadoop clusters, the core-site.xml file is world-readable. To protect
+these credentials from prying eyes, it is recommended that you use the
+credential provider framework to securely store them and access them through
+configuration.
+
+All ADLS credential properties can be protected by credential providers.
+For additional reading on the credential provider API, see
+[Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html).
+
+#### Provisioning
+
+```
+% hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+    -provider localjceks://file/home/foo/adls.jceks
+% hadoop credential create dfs.adls.oauth2.credential -value 123
+    -provider localjceks://file/home/foo/adls.jceks
+```
+
+#### Configuring core-site.xml or command line property
+
+```
+<property>
+  <name>hadoop.security.credential.provider.path</name>
+  <value>localjceks://file/home/foo/adls.jceks</value>
+  <description>Path to interrogate for protected credentials.</description>
+</property>
+```
+
+#### Running DistCp
+
+```
+% hadoop distcp
+    [-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
+    hdfs://<NameNode Hostname>:9001/user/foo/007020615
+    adl://<Account Name>.azuredatalakestore.net/testDir/
+```
+
+NOTE: You may optionally add the provider path property to the distcp command
+line instead of added job specific configuration to a generic core-site.xml.
+The square brackets above illustrate this capability.
+
 
 ## <a name="Enabling_ADL" />Enabling ADL Filesystem
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01624f0c/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index c94e692..70f2a7f 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -18,10 +18,12 @@
 
 package org.apache.hadoop.fs.adl;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 
+import org.apache.commons.lang.builder.EqualsBuilder;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
 import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
@@ -39,15 +41,29 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
 import static org.apache.hadoop.fs.adl.TokenProviderType.*;
+import static org.junit.Assert.assertEquals;
 
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 /**
  * Test appropriate token provider is loaded as per configuration.
  */
 public class TestAzureADTokenProvider {
 
+  private static final String CLIENT_ID = "MY_CLIENT_ID";
+  private static final String REFRESH_TOKEN = "MY_REFRESH_TOKEN";
+  private static final String CLIENT_SECRET = "MY_CLIENT_SECRET";
+  private static final String REFRESH_URL = "http://localhost:8080/refresh";
+
+  @Rule
+  public final TemporaryFolder tempDir = new TemporaryFolder();
+
   @Test
   public void testRefreshTokenProvider()
       throws URISyntaxException, IOException {
@@ -130,4 +146,140 @@ public class TestAzureADTokenProvider {
           e.getMessage().contains("wrong.classpath.CustomMockTokenProvider"));
     }
   }
+
+  private CredentialProvider createTempCredProvider(Configuration conf)
+      throws URISyntaxException, IOException {
+    final File file = tempDir.newFile("test.jks");
+    final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+        file.toURI());
+    conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+        jks.toString());
+    return CredentialProviderFactory.getProviders(conf).get(0);
+  }
+
+  @Test
+  public void testRefreshTokenWithCredentialProvider()
+      throws IOException, URISyntaxException {
+    Configuration conf = new Configuration();
+    conf.set(AZURE_AD_CLIENT_ID_KEY, "DUMMY");
+    conf.set(AZURE_AD_REFRESH_TOKEN_KEY, "DUMMY");
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, RefreshToken);
+
+    CredentialProvider provider = createTempCredProvider(conf);
+    provider.createCredentialEntry(AZURE_AD_CLIENT_ID_KEY,
+        CLIENT_ID.toCharArray());
+    provider.createCredentialEntry(AZURE_AD_REFRESH_TOKEN_KEY,
+        REFRESH_TOKEN.toCharArray());
+    provider.flush();
+
+    URI uri = new URI("adl://localhost:8080");
+    AdlFileSystem fileSystem = new AdlFileSystem();
+    fileSystem.initialize(uri, conf);
+    RefreshTokenBasedTokenProvider expected =
+        new RefreshTokenBasedTokenProvider(CLIENT_ID, REFRESH_TOKEN);
+    Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+        fileSystem.getTokenProvider()));
+  }
+
+  @Test
+  public void testRefreshTokenWithCredentialProviderFallback()
+      throws IOException, URISyntaxException {
+    Configuration conf = new Configuration();
+    conf.set(AZURE_AD_CLIENT_ID_KEY, CLIENT_ID);
+    conf.set(AZURE_AD_REFRESH_TOKEN_KEY, REFRESH_TOKEN);
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, RefreshToken);
+
+    createTempCredProvider(conf);
+
+    URI uri = new URI("adl://localhost:8080");
+    AdlFileSystem fileSystem = new AdlFileSystem();
+    fileSystem.initialize(uri, conf);
+    RefreshTokenBasedTokenProvider expected =
+        new RefreshTokenBasedTokenProvider(CLIENT_ID, REFRESH_TOKEN);
+    Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+        fileSystem.getTokenProvider()));
+  }
+
+  @Test
+  public void testClientCredWithCredentialProvider()
+      throws IOException, URISyntaxException {
+    Configuration conf = new Configuration();
+    conf.set(AZURE_AD_CLIENT_ID_KEY, "DUMMY");
+    conf.set(AZURE_AD_CLIENT_SECRET_KEY, "DUMMY");
+    conf.set(AZURE_AD_REFRESH_URL_KEY, "DUMMY");
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, ClientCredential);
+
+    CredentialProvider provider = createTempCredProvider(conf);
+    provider.createCredentialEntry(AZURE_AD_CLIENT_ID_KEY,
+        CLIENT_ID.toCharArray());
+    provider.createCredentialEntry(AZURE_AD_CLIENT_SECRET_KEY,
+        CLIENT_SECRET.toCharArray());
+    provider.createCredentialEntry(AZURE_AD_REFRESH_URL_KEY,
+        REFRESH_URL.toCharArray());
+    provider.flush();
+
+    URI uri = new URI("adl://localhost:8080");
+    AdlFileSystem fileSystem = new AdlFileSystem();
+    fileSystem.initialize(uri, conf);
+    ClientCredsTokenProvider expected = new ClientCredsTokenProvider(
+        REFRESH_URL, CLIENT_ID, CLIENT_SECRET);
+    Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+        fileSystem.getTokenProvider()));
+  }
+
+  @Test
+  public void testClientCredWithCredentialProviderFallback()
+      throws IOException, URISyntaxException {
+    Configuration conf = new Configuration();
+    conf.set(AZURE_AD_CLIENT_ID_KEY, CLIENT_ID);
+    conf.set(AZURE_AD_CLIENT_SECRET_KEY, CLIENT_SECRET);
+    conf.set(AZURE_AD_REFRESH_URL_KEY, REFRESH_URL);
+    conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, ClientCredential);
+
+    createTempCredProvider(conf);
+
+    URI uri = new URI("adl://localhost:8080");
+    AdlFileSystem fileSystem = new AdlFileSystem();
+    fileSystem.initialize(uri, conf);
+    ClientCredsTokenProvider expected = new ClientCredsTokenProvider(
+        REFRESH_URL, CLIENT_ID, CLIENT_SECRET);
+    Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+        fileSystem.getTokenProvider()));
+  }
+
+  @Test
+  public void testCredentialProviderPathExclusions() throws Exception {
+    String providerPath =
+        "user:///,jceks://adl/user/hrt_qa/sqoopdbpasswd.jceks," +
+            "jceks://hdfs@nn1.example.com/my/path/test.jceks";
+    Configuration config = new Configuration();
+    config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+        providerPath);
+    String newPath =
+        "user:///,jceks://hdfs@nn1.example.com/my/path/test.jceks";
+
+    excludeAndTestExpectations(config, newPath);
+  }
+
+  @Test
+  public void testExcludeAllProviderTypesFromConfig() throws Exception {
+    String providerPath =
+        "jceks://adl/tmp/test.jceks," +
+            "jceks://adl@/my/path/test.jceks";
+    Configuration config = new Configuration();
+    config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+        providerPath);
+    String newPath = null;
+
+    excludeAndTestExpectations(config, newPath);
+  }
+
+  void excludeAndTestExpectations(Configuration config, String newPath)
+      throws Exception {
+    Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
+        config, AdlFileSystem.class);
+    String effectivePath = conf.get(
+        CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, null);
+    assertEquals(newPath, effectivePath);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message