hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tmarqua...@apache.org
Subject [32/50] [abbrv] hadoop git commit: HADOOP-15694. ABFS: Allow OAuth credentials to not be tied to accounts. Contributed by Sean Mackrory.
Date Mon, 17 Sep 2018 22:01:54 GMT
HADOOP-15694. ABFS: Allow OAuth credentials to not be tied to accounts.
Contributed by Sean Mackrory.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5593cbd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5593cbd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5593cbd

Branch: refs/heads/HADOOP-15407
Commit: e5593cbd8323399359b3e8da46bd58e8364cbf22
Parents: 13c70e9
Author: Thomas Marquardt <tmarq@microsoft.com>
Authored: Wed Sep 12 22:51:41 2018 +0000
Committer: Thomas Marquardt <tmarq@microsoft.com>
Committed: Mon Sep 17 19:54:01 2018 +0000

----------------------------------------------------------------------
 .../hadoop/fs/azurebfs/AbfsConfiguration.java   | 208 ++++++++++----
 .../fs/azurebfs/AzureBlobFileSystemStore.java   |  69 ++---
 .../azurebfs/constants/ConfigurationKeys.java   |  44 +--
 .../oauth2/UserPasswordTokenProvider.java       |  10 -
 .../services/ShellDecryptionKeyProvider.java    |  14 +-
 .../fs/azurebfs/services/SimpleKeyProvider.java |  18 +-
 .../hadoop-azure/src/site/markdown/abfs.md      |  10 +
 .../azurebfs/AbstractAbfsIntegrationTest.java   |  63 +++--
 .../fs/azurebfs/AbstractAbfsScaleTest.java      |   8 +-
 .../hadoop/fs/azurebfs/ITestAbfsClient.java     |   6 +-
 .../fs/azurebfs/ITestAbfsReadWriteAndSeek.java  |   4 +-
 .../ITestAzureBlobFileSystemAppend.java         |   3 +-
 .../ITestAzureBlobFileSystemBackCompat.java     |   3 +-
 .../azurebfs/ITestAzureBlobFileSystemCopy.java  |   3 +-
 .../ITestAzureBlobFileSystemCreate.java         |   3 +-
 .../ITestAzureBlobFileSystemDelete.java         |   3 +-
 .../azurebfs/ITestAzureBlobFileSystemE2E.java   |   5 +-
 .../ITestAzureBlobFileSystemE2EScale.java       |   2 +-
 .../ITestAzureBlobFileSystemFileStatus.java     |   2 +-
 .../ITestAzureBlobFileSystemFinalize.java       |   8 +-
 .../azurebfs/ITestAzureBlobFileSystemFlush.java |   2 +-
 .../ITestAzureBlobFileSystemInitAndCreate.java  |   2 +-
 .../ITestAzureBlobFileSystemListStatus.java     |   3 +-
 .../azurebfs/ITestAzureBlobFileSystemMkDir.java |   3 +-
 .../azurebfs/ITestAzureBlobFileSystemOauth.java |  20 +-
 .../ITestAzureBlobFileSystemRename.java         |   4 +-
 .../ITestAzureBlobFileSystemRenameUnicode.java  |   2 +-
 .../azurebfs/ITestFileSystemInitialization.java |   8 +-
 .../fs/azurebfs/ITestFileSystemProperties.java  |   2 +-
 .../azurebfs/ITestFileSystemRegistration.java   |  15 +-
 .../TestAbfsConfigurationFieldsValidation.java  |  24 +-
 .../fs/azurebfs/TestAccountConfiguration.java   | 273 +++++++++++++++++++
 .../constants/TestConfigurationKeys.java        |   2 +-
 .../contract/ABFSContractTestBinding.java       |   8 +-
 .../ITestAbfsFileSystemContractAppend.java      |   2 +-
 .../ITestAbfsFileSystemContractConcat.java      |   2 +-
 .../ITestAbfsFileSystemContractCreate.java      |   2 +-
 .../ITestAbfsFileSystemContractDelete.java      |   2 +-
 ...TestAbfsFileSystemContractGetFileStatus.java |   2 +-
 .../ITestAbfsFileSystemContractMkdir.java       |   2 +-
 .../ITestAbfsFileSystemContractOpen.java        |   2 +-
 .../ITestAbfsFileSystemContractRename.java      |   2 +-
 ...TestAbfsFileSystemContractRootDirectory.java |   2 +-
 .../ITestAbfsFileSystemContractSeek.java        |   2 +-
 .../ITestAbfsFileSystemContractSetTimes.java    |   2 +-
 .../fs/azurebfs/services/TestAbfsClient.java    |  10 +-
 .../TestShellDecryptionKeyProvider.java         |   1 +
 .../hadoop/fs/azurebfs/utils/AbfsTestUtils.java |   5 +
 48 files changed, 665 insertions(+), 227 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 518fef9..927a315 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.apache.hadoop.fs.azurebfs.services.KeyProvider;
 import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider;
 import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx;
+import org.apache.hadoop.security.ProviderUtils;
 import org.apache.hadoop.util.ReflectionUtils;
 
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
@@ -65,7 +66,8 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class AbfsConfiguration{
-  private final Configuration configuration;
+  private final Configuration rawConfig;
+  private final String accountName;
   private final boolean isSecure;
 
   @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE,
@@ -155,9 +157,12 @@ public class AbfsConfiguration{
 
   private Map<String, String> storageAccountKeys;
 
-  public AbfsConfiguration(final Configuration configuration) throws IllegalAccessException, InvalidConfigurationValueException {
-    this.configuration = configuration;
-    this.isSecure = this.configuration.getBoolean(FS_AZURE_SECURE_MODE, false);
+  public AbfsConfiguration(final Configuration rawConfig, String accountName)
+      throws IllegalAccessException, InvalidConfigurationValueException, IOException {
+    this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders(
+        rawConfig, AzureBlobFileSystem.class);
+    this.accountName = accountName;
+    this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false);
 
     validateStorageAccountKeys();
     Field[] fields = this.getClass().getDeclaredFields();
@@ -177,14 +182,130 @@ public class AbfsConfiguration{
     }
   }
 
+  /**
+   * Appends an account name to a configuration key yielding the
+   * account-specific form.
+   * @param key Account-agnostic configuration key
+   * @return Account-specific configuration key
+   */
+  public String accountConf(String key) {
+    return key + "." + accountName;
+  }
+
+  /**
+   * Returns the account-specific value if it exists, then looks for an
+   * account-agnostic value.
+   * @param key Account-agnostic configuration key
+   * @return value if one exists, else null
+   */
+  public String get(String key) {
+    return rawConfig.get(accountConf(key), rawConfig.get(key));
+  }
+
+  /**
+   * Returns the account-specific value if it exists, then looks for an
+   * account-agnostic value, and finally tries the default value.
+   * @param key Account-agnostic configuration key
+   * @param defaultValue Value returned if none is configured
+   * @return value if one exists, else the default value
+   */
+  public boolean getBoolean(String key, boolean defaultValue) {
+    return rawConfig.getBoolean(accountConf(key), rawConfig.getBoolean(key, defaultValue));
+  }
+
+  /**
+   * Returns the account-specific value if it exists, then looks for an
+   * account-agnostic value, and finally tries the default value.
+   * @param key Account-agnostic configuration key
+   * @param defaultValue Value returned if none is configured
+   * @return value if one exists, else the default value
+   */
+  public long getLong(String key, long defaultValue) {
+    return rawConfig.getLong(accountConf(key), rawConfig.getLong(key, defaultValue));
+  }
+
+  /**
+   * Returns the account-specific password in string form if it exists, then
+   * looks for an account-agnostic value.
+   * @param key Account-agnostic configuration key
+   * @return value in String form if one exists, else null
+   * @throws IOException
+   */
+  public String getPasswordString(String key) throws IOException {
+    char[] passchars = rawConfig.getPassword(accountConf(key));
+    if (passchars == null) {
+      passchars = rawConfig.getPassword(key);
+    }
+    if (passchars != null) {
+      return new String(passchars);
+    }
+    return null;
+  }
+
+  /**
+   * Returns the account-specific Class if it exists, then looks for an
+   * account-agnostic value, and finally tries the default value.
+   * @param name Account-agnostic configuration key
+   * @param defaultValue Class returned if none is configured
+   * @param xface Interface shared by all possible values
+   * @return Highest-precedence Class object that was found
+   * @throws IOException
+   */
+  public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) {
+    return rawConfig.getClass(accountConf(name),
+        rawConfig.getClass(name, defaultValue, xface),
+        xface);
+  }
+
+  /**
+   * Returns the account-specific password in string form if it exists, then
+   * looks for an account-agnostic value.
+   * @param name Account-agnostic configuration key
+   * @param defaultValue Value returned if none is configured
+   * @return value in String form if one exists, else null
+   * @throws IOException
+   */
+  public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
+    return rawConfig.getEnum(accountConf(name),
+        rawConfig.getEnum(name, defaultValue));
+  }
+
+  /**
+   * Unsets parameter in the underlying Configuration object.
+   * Provided only as a convenience; does not add any account logic.
+   * @param key Configuration key
+   */
+  public void unset(String key) {
+    rawConfig.unset(key);
+  }
+
+  /**
+   * Sets String in the underlying Configuration object.
+   * Provided only as a convenience; does not add any account logic.
+   * @param key Configuration key
+   * @param value Configuration value
+   */
+  public void set(String key, String value) {
+    rawConfig.set(key, value);
+  }
+
+  /**
+   * Sets boolean in the underlying Configuration object.
+   * Provided only as a convenience; does not add any account logic.
+   * @param key Configuration key
+   * @param value Configuration value
+   */
+  public void setBoolean(String key, boolean value) {
+    rawConfig.setBoolean(key, value);
+  }
+
   public boolean isSecureMode() {
     return isSecure;
   }
 
-  public String getStorageAccountKey(final String accountName) throws AzureBlobFileSystemException {
+  public String getStorageAccountKey() throws AzureBlobFileSystemException {
     String key;
-    String keyProviderClass =
-            configuration.get(AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX + accountName);
+    String keyProviderClass = get(AZURE_KEY_ACCOUNT_KEYPROVIDER);
     KeyProvider keyProvider;
 
     if (keyProviderClass == null) {
@@ -195,7 +316,7 @@ public class AbfsConfiguration{
       // implements KeyProvider
       Object keyProviderObject;
       try {
-        Class<?> clazz = configuration.getClassByName(keyProviderClass);
+        Class<?> clazz = rawConfig.getClassByName(keyProviderClass);
         keyProviderObject = clazz.newInstance();
       } catch (Exception e) {
         throw new KeyProviderException("Unable to load key provider class.", e);
@@ -206,7 +327,7 @@ public class AbfsConfiguration{
       }
       keyProvider = (KeyProvider) keyProviderObject;
     }
-    key = keyProvider.getStorageAccountKey(accountName, configuration);
+    key = keyProvider.getStorageAccountKey(accountName, rawConfig);
 
     if (key == null) {
       throw new ConfigurationPropertyNotFoundException(accountName);
@@ -215,8 +336,8 @@ public class AbfsConfiguration{
     return key;
   }
 
-  public Configuration getConfiguration() {
-    return this.configuration;
+  public Configuration getRawConfiguration() {
+    return this.rawConfig;
   }
 
   public int getWriteBufferSize() {
@@ -292,11 +413,11 @@ public class AbfsConfiguration{
   }
 
   public SSLSocketFactoryEx.SSLChannelMode getPreferredSSLFactoryOption() {
-    return configuration.getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE);
+    return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE);
   }
 
-  public AuthType getAuthType(final String accountName) {
-    return configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey);
+  public AuthType getAuthType(String accountName) {
+    return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey);
   }
 
   public boolean isDelegationTokenManagerEnabled() {
@@ -304,34 +425,34 @@ public class AbfsConfiguration{
   }
 
   public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException {
-    return new AbfsDelegationTokenManager(configuration);
+    return new AbfsDelegationTokenManager(getRawConfiguration());
   }
 
-  public AccessTokenProvider getTokenProvider(final String accountName) throws TokenAccessProviderException {
-    AuthType authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey);
+  public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException {
+    AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey);
     if (authType == AuthType.OAuth) {
       try {
         Class<? extends AccessTokenProvider> tokenProviderClass =
-                configuration.getClass(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName, null,
+                getClass(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null,
                         AccessTokenProvider.class);
         AccessTokenProvider tokenProvider = null;
         if (tokenProviderClass == ClientCredsTokenProvider.class) {
-          String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + accountName);
-          String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountName);
-          String clientSecret = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + accountName);
+          String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT);
+          String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
+          String clientSecret = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET);
           tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId, clientSecret);
         } else if (tokenProviderClass == UserPasswordTokenProvider.class) {
-          String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + accountName);
-          String username = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME + accountName);
-          String password = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD + accountName);
+          String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT);
+          String username = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME);
+          String password = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD);
           tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password);
         } else if (tokenProviderClass == MsiTokenProvider.class) {
-          String tenantGuid = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT + accountName);
-          String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountName);
+          String tenantGuid = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT);
+          String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
           tokenProvider = new MsiTokenProvider(tenantGuid, clientId);
         } else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) {
-          String refreshToken = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN + accountName);
-          String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountName);
+          String refreshToken = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN);
+          String clientId = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
           tokenProvider = new RefreshTokenBasedTokenProvider(clientId, refreshToken);
         } else {
           throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass);
@@ -345,20 +466,19 @@ public class AbfsConfiguration{
 
     } else if (authType == AuthType.Custom) {
       try {
-        String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName;
+        String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
         Class<? extends CustomTokenProviderAdaptee> customTokenProviderClass =
-                configuration.getClass(configKey, null,
-                        CustomTokenProviderAdaptee.class);
+                getClass(configKey, null, CustomTokenProviderAdaptee.class);
         if (customTokenProviderClass == null) {
           throw new IllegalArgumentException(
                   String.format("The configuration value for \"%s\" is invalid.", configKey));
         }
         CustomTokenProviderAdaptee azureTokenProvider = ReflectionUtils
-                .newInstance(customTokenProviderClass, configuration);
+                .newInstance(customTokenProviderClass, rawConfig);
         if (azureTokenProvider == null) {
           throw new IllegalArgumentException("Failed to initialize " + customTokenProviderClass);
         }
-        azureTokenProvider.initialize(configuration, accountName);
+        azureTokenProvider.initialize(rawConfig, accountName);
         return new CustomTokenProviderAdapter(azureTokenProvider);
       } catch(IllegalArgumentException e) {
         throw e;
@@ -375,7 +495,7 @@ public class AbfsConfiguration{
   void validateStorageAccountKeys() throws InvalidConfigurationValueException {
     Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator(
         FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true);
-    this.storageAccountKeys = configuration.getValByRegex(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX);
+    this.storageAccountKeys = rawConfig.getValByRegex(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX);
 
     for (Map.Entry<String, String> account : storageAccountKeys.entrySet()) {
       validator.validate(account.getValue());
@@ -384,7 +504,7 @@ public class AbfsConfiguration{
 
   int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
     IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class);
-    String value = configuration.get(validator.ConfigurationKey());
+    String value = get(validator.ConfigurationKey());
 
     // validate
     return new IntegerConfigurationBasicValidator(
@@ -397,7 +517,7 @@ public class AbfsConfiguration{
 
   long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
     LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class);
-    String value = configuration.get(validator.ConfigurationKey());
+    String value = rawConfig.get(validator.ConfigurationKey());
 
     // validate
     return new LongConfigurationBasicValidator(
@@ -410,7 +530,7 @@ public class AbfsConfiguration{
 
   String validateString(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
     StringConfigurationValidatorAnnotation validator = field.getAnnotation(StringConfigurationValidatorAnnotation.class);
-    String value = configuration.get(validator.ConfigurationKey());
+    String value = rawConfig.get(validator.ConfigurationKey());
 
     // validate
     return new StringConfigurationBasicValidator(
@@ -421,7 +541,7 @@ public class AbfsConfiguration{
 
   String validateBase64String(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
     Base64StringConfigurationValidatorAnnotation validator = field.getAnnotation((Base64StringConfigurationValidatorAnnotation.class));
-    String value = configuration.get(validator.ConfigurationKey());
+    String value = rawConfig.get(validator.ConfigurationKey());
 
     // validate
     return new Base64StringConfigurationBasicValidator(
@@ -432,7 +552,7 @@ public class AbfsConfiguration{
 
   boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
     BooleanConfigurationValidatorAnnotation validator = field.getAnnotation(BooleanConfigurationValidatorAnnotation.class);
-    String value = configuration.get(validator.ConfigurationKey());
+    String value = rawConfig.get(validator.ConfigurationKey());
 
     // validate
     return new BooleanConfigurationBasicValidator(
@@ -441,14 +561,6 @@ public class AbfsConfiguration{
         validator.ThrowIfInvalid()).validate(value);
   }
 
-  String getPasswordString(String key) throws IOException {
-    char[] passchars = configuration.getPassword(key);
-    if (passchars != null) {
-      return new String(passchars);
-    }
-    return null;
-  }
-
   @VisibleForTesting
   void setReadBufferSize(int bufferSize) {
     this.readBufferSize = bufferSize;
@@ -463,4 +575,4 @@ public class AbfsConfiguration{
   void setEnableFlush(boolean enableFlush) {
     this.enableFlush = enableFlush;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 6542a64..e3bea32 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -113,10 +113,15 @@ public class AzureBlobFileSystemStore {
   private boolean isNamespaceEnabled;
 
   public AzureBlobFileSystemStore(URI uri, boolean isSecure, Configuration configuration, UserGroupInformation userGroupInformation)
-          throws AzureBlobFileSystemException {
+          throws AzureBlobFileSystemException, IOException {
     this.uri = uri;
+
+    String[] authorityParts = authorityParts(uri);
+    final String fileSystemName = authorityParts[0];
+    final String accountName = authorityParts[1];
+
     try {
-      this.abfsConfiguration = new AbfsConfiguration(configuration);
+      this.abfsConfiguration = new AbfsConfiguration(configuration, accountName);
     } catch (IllegalAccessException exception) {
       throw new FileSystemOperationUnhandledException(exception);
     }
@@ -125,7 +130,31 @@ public class AzureBlobFileSystemStore {
     this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList(
         abfsConfiguration.getAzureAtomicRenameDirs().split(AbfsHttpConstants.COMMA)));
 
-    initializeClient(uri, isSecure);
+    initializeClient(uri, fileSystemName, accountName, isSecure);
+  }
+
+  private String[] authorityParts(URI uri) throws InvalidUriAuthorityException, InvalidUriException {
+    final String authority = uri.getRawAuthority();
+    if (null == authority) {
+      throw new InvalidUriAuthorityException(uri.toString());
+    }
+
+    if (!authority.contains(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER)) {
+      throw new InvalidUriAuthorityException(uri.toString());
+    }
+
+    final String[] authorityParts = authority.split(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER, 2);
+
+    if (authorityParts.length < 2 || authorityParts[0] != null
+        && authorityParts[0].isEmpty()) {
+      final String errMsg = String
+              .format("'%s' has a malformed authority, expected container name. "
+                      + "Authority takes the form "
+                      + FileSystemUriSchemes.ABFS_SCHEME + "://[<container name>@]<account name>",
+                      uri.toString());
+      throw new InvalidUriException(errMsg);
+    }
+    return authorityParts;
   }
 
   public boolean getIsNamespaceEnabled() throws AzureBlobFileSystemException {
@@ -154,7 +183,7 @@ public class AzureBlobFileSystemStore {
     // the Azure Storage Service URI changes from
     // http[s]://[account][domain-suffix]/[filesystem] to
     // http[s]://[ip]:[port]/[account]/[filesystem].
-    String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT);
+    String endPoint = abfsConfiguration.get(AZURE_ABFS_ENDPOINT);
     if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) {
       uriBuilder.setHost(hostName);
       return uriBuilder;
@@ -738,36 +767,12 @@ public class AzureBlobFileSystemStore {
     return isKeyForDirectorySet(key, azureAtomicRenameDirSet);
   }
 
-  private void initializeClient(URI uri, boolean isSeure) throws AzureBlobFileSystemException {
+  private void initializeClient(URI uri, String fileSystemName, String accountName, boolean isSecure) throws AzureBlobFileSystemException {
     if (this.client != null) {
       return;
     }
 
-    final String authority = uri.getRawAuthority();
-    if (null == authority) {
-      throw new InvalidUriAuthorityException(uri.toString());
-    }
-
-    if (!authority.contains(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER)) {
-      throw new InvalidUriAuthorityException(uri.toString());
-    }
-
-    final String[] authorityParts = authority.split(AbfsHttpConstants.AZURE_DISTRIBUTED_FILE_SYSTEM_AUTHORITY_DELIMITER, 2);
-
-    if (authorityParts.length < 2 || authorityParts[0] != null
-        && authorityParts[0].isEmpty()) {
-      final String errMsg = String
-              .format("'%s' has a malformed authority, expected container name. "
-                      + "Authority takes the form "
-                      + FileSystemUriSchemes.ABFS_SCHEME + "://[<container name>@]<account name>",
-                      uri.toString());
-      throw new InvalidUriException(errMsg);
-    }
-
-    final String fileSystemName = authorityParts[0];
-    final String accountName = authorityParts[1];
-
-    final URIBuilder uriBuilder = getURIBuilder(accountName, isSeure);
+    final URIBuilder uriBuilder = getURIBuilder(accountName, isSecure);
 
     final String url = uriBuilder.toString() + AbfsHttpConstants.FORWARD_SLASH + fileSystemName;
 
@@ -788,9 +793,9 @@ public class AzureBlobFileSystemStore {
                 uri.toString() + " - account name is not fully qualified.");
       }
       creds = new SharedKeyCredentials(accountName.substring(0, dotIndex),
-            abfsConfiguration.getStorageAccountKey(accountName));
+            abfsConfiguration.getStorageAccountKey());
     } else {
-      tokenProvider = abfsConfiguration.getTokenProvider(accountName);
+      tokenProvider = abfsConfiguration.getTokenProvider();
     }
 
     this.client =  new AbfsClient(baseUrl, creds, abfsConfiguration, new ExponentialRetryPolicy(), tokenProvider);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 5236719..13cdaeb 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public final class ConfigurationKeys {
-  public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+  public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key";
   public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)";
   public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode";
 
@@ -54,29 +54,33 @@ public final class ConfigurationKeys {
   public static final String FS_AZURE_USER_AGENT_PREFIX_KEY = "fs.azure.user.agent.prefix";
   public static final String FS_AZURE_SSL_CHANNEL_MODE_KEY = "fs.azure.ssl.channel.mode";
 
-  public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider.";
+  public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER = "fs.azure.account.keyprovider";
   public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
 
   /** End point of ABFS account: {@value}. */
   public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint";
-  /** Prefix for auth type properties: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type.";
-  /** Prefix for oauth token provider type: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME = "fs.azure.account.oauth.provider.type.";
-  /** Prefix for oauth AAD client id: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID = "fs.azure.account.oauth2.client.id.";
-  /** Prefix for oauth AAD client secret: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret.";
-  /** Prefix for oauth AAD client endpoint: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT = "fs.azure.account.oauth2.client.endpoint.";
-  /** Prefix for oauth msi tenant id: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT = "fs.azure.account.oauth2.msi.tenant.";
-  /** Prefix for oauth user name: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_USER_NAME = "fs.azure.account.oauth2.user.name.";
-  /** Prefix for oauth user password: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD = "fs.azure.account.oauth2.user.password.";
-  /** Prefix for oauth refresh token: {@value}. */
-  public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token.";
+  /** Key for auth type properties: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type";
+  /** Key for oauth token provider type: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME = "fs.azure.account.oauth.provider.type";
+  /** Key for oauth AAD client id: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID = "fs.azure.account.oauth2.client.id";
+  /** Key for oauth AAD client secret: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET = "fs.azure.account.oauth2.client.secret";
+  /** Key for oauth AAD client endpoint: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT = "fs.azure.account.oauth2.client.endpoint";
+  /** Key for oauth msi tenant id: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT = "fs.azure.account.oauth2.msi.tenant";
+  /** Key for oauth user name: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_USER_NAME = "fs.azure.account.oauth2.user.name";
+  /** Key for oauth user password: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD = "fs.azure.account.oauth2.user.password";
+  /** Key for oauth refresh token: {@value}. */
+  public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token";
+
+  public static String accountProperty(String property, String account) {
+    return property + "." + account;
+  }
 
   public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token";
   public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
index 7504e9d..3dad32e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
@@ -24,7 +24,6 @@ import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.conf.Configuration;
 /**
  * Provides tokens based on username and password.
  */
@@ -54,13 +53,4 @@ public class UserPasswordTokenProvider extends AccessTokenProvider {
     LOG.debug("AADToken: refreshing user-password based token");
     return AzureADAuthenticator.getTokenUsingClientCreds(authEndpoint, username, password);
   }
-
-  private static String getPasswordString(Configuration conf, String key)
-          throws IOException {
-    char[] passchars = conf.getPassword(key);
-    if (passchars == null) {
-      throw new IOException("Password " + key + " not found");
-    }
-    return new String(passchars);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java
index 3fc05ff..bdac922 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ShellDecryptionKeyProvider.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
 import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException;
 import org.apache.hadoop.util.Shell;
@@ -36,11 +37,18 @@ public class ShellDecryptionKeyProvider extends SimpleKeyProvider {
   private static final Logger LOG = LoggerFactory.getLogger(ShellDecryptionKeyProvider.class);
 
   @Override
-  public String getStorageAccountKey(String accountName, Configuration conf)
+  public String getStorageAccountKey(String accountName, Configuration rawConfig)
       throws KeyProviderException {
-    String envelope = super.getStorageAccountKey(accountName, conf);
+    String envelope = super.getStorageAccountKey(accountName, rawConfig);
 
-    final String command = conf.get(ConfigurationKeys.AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT);
+    AbfsConfiguration abfsConfig;
+    try {
+      abfsConfig = new AbfsConfiguration(rawConfig, accountName);
+    } catch(IllegalAccessException | IOException e) {
+      throw new KeyProviderException("Unable to get key from credential providers.", e);
+    }
+
+    final String command = abfsConfig.get(ConfigurationKeys.AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT);
     if (command == null) {
       throw new KeyProviderException(
           "Script path is not specified via fs.azure.shellkeyprovider.script");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
index cedae57..727e1b3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
@@ -21,10 +21,10 @@ package org.apache.hadoop.fs.azurebfs.services;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
 import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException;
-import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,19 +36,19 @@ public class SimpleKeyProvider implements KeyProvider {
   private static final Logger LOG = LoggerFactory.getLogger(SimpleKeyProvider.class);
 
   @Override
-  public String getStorageAccountKey(String accountName, Configuration conf)
+  public String getStorageAccountKey(String accountName, Configuration rawConfig)
       throws KeyProviderException {
     String key = null;
+
     try {
-      Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
-          conf, AzureBlobFileSystem.class);
-      char[] keyChars = c.getPassword(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + accountName);
-      if (keyChars != null) {
-        key = new String(keyChars);
-      }
+      AbfsConfiguration abfsConfig = new AbfsConfiguration(rawConfig, accountName);
+      key = abfsConfig.getPasswordString(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME);
+    } catch(IllegalAccessException | InvalidConfigurationValueException e) {
+      throw new KeyProviderException("Failure to initialize configuration", e);
     } catch(IOException ioe) {
       LOG.warn("Unable to get key from credential providers. {}", ioe);
     }
+
     return key;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index a4b3483..db55e67 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -62,6 +62,16 @@ The abfs client has a fully consistent view of the store, which has complete Cre
 * Directory Rename: `O(files)`.
 * Directory Delete: `O(files)`.
 
+## Configuring ABFS
+
+Any configuration can be specified generally (or as the default when accessing all accounts) or can be tied to s a specific account.
+For example, an OAuth identity can be configured for use regardless of which account is accessed with the property
+"fs.azure.account.oauth2.client.id"
+or you can configure an identity to be used only for a specific storage account with
+"fs.azure.account.oauth2.client.id.\<account\_name\>.dfs.core.windows.net".
+
+Note that it doesn't make sense to do this with some properties, like shared keys that are inherently account-specific.
+
 ## Testing ABFS
 
 See the relevant section in [Testing Azure](testing_azure.html).

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index f61954f..52185cd 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -67,38 +67,39 @@ public abstract class AbstractAbfsIntegrationTest extends
   private AzureBlobFileSystem abfs;
   private String abfsScheme;
 
-  private Configuration configuration;
+  private Configuration rawConfig;
+  private AbfsConfiguration abfsConfig;
   private String fileSystemName;
   private String accountName;
   private String testUrl;
   private AuthType authType;
 
-  protected AbstractAbfsIntegrationTest() {
+  protected AbstractAbfsIntegrationTest() throws Exception {
     fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
-    configuration = new Configuration();
-    configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+    rawConfig = new Configuration();
+    rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
 
-    this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME);
+    this.accountName = rawConfig.get(FS_AZURE_ACCOUNT_NAME);
     if (accountName == null) {
       // check if accountName is set using different config key
-      accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME);
+      accountName = rawConfig.get(FS_AZURE_ABFS_ACCOUNT_NAME);
     }
     assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME,
             accountName != null && !accountName.isEmpty());
 
+    abfsConfig = new AbfsConfiguration(rawConfig, accountName);
 
-    authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME
-            + accountName, AuthType.SharedKey);
+    authType = abfsConfig.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey);
     abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME
             : FileSystemUriSchemes.ABFS_SECURE_SCHEME;
 
     if (authType == AuthType.SharedKey) {
-      String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName;
-      assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null);
+      assumeTrue("Not set: " + FS_AZURE_ACCOUNT_KEY,
+          abfsConfig.get(FS_AZURE_ACCOUNT_KEY) != null);
       // Update credentials
     } else {
-      String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName;
-      assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null);
+      assumeTrue("Not set: " + FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME,
+          abfsConfig.get(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME) != null);
     }
 
     final String abfsUrl = this.getFileSystemName() + "@" + this.getAccountName();
@@ -111,14 +112,14 @@ public abstract class AbstractAbfsIntegrationTest extends
     }
 
     this.testUrl = defaultUri.toString();
-    configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
-    configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
+    abfsConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
+    abfsConfig.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
     // For testing purposes, an IP address and port may be provided to override
     // the host specified in the FileSystem URI.  Also note that the format of
     // the Azure Storage Service URI changes from
     // http[s]://[account][domain-suffix]/[filesystem] to
     // http[s]://[ip]:[port]/[account]/[filesystem].
-    String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
+    String endPoint = abfsConfig.get(AZURE_ABFS_ENDPOINT);
     if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) {
       this.isIPAddress = true;
     } else {
@@ -140,18 +141,18 @@ public abstract class AbstractAbfsIntegrationTest extends
       // update configuration with wasb credentials
       String accountNameWithoutDomain = accountName.split("\\.")[0];
       String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
-      String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName;
-      if (configuration.get(keyProperty) == null) {
-        configuration.set(keyProperty, getAccountKey());
+      String keyProperty = FS_AZURE_ACCOUNT_KEY + "." + wasbAccountName;
+      if (rawConfig.get(keyProperty) == null) {
+        rawConfig.set(keyProperty, getAccountKey());
       }
 
       azureNativeFileSystemStore.initialize(
           wasbUri,
-          configuration,
-          new AzureFileSystemInstrumentation(getConfiguration()));
+          rawConfig,
+          new AzureFileSystemInstrumentation(rawConfig));
 
       wasb = new NativeAzureFileSystem(azureNativeFileSystemStore);
-      wasb.initialize(wasbUri, configuration);
+      wasb.initialize(wasbUri, rawConfig);
     }
   }
 
@@ -197,8 +198,8 @@ public abstract class AbstractAbfsIntegrationTest extends
   }
 
   public AzureBlobFileSystem getFileSystem(String abfsUri) throws Exception {
-    configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, abfsUri);
-    final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(configuration);
+    abfsConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, abfsUri);
+    final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig);
     return fs;
   }
 
@@ -210,7 +211,7 @@ public abstract class AbstractAbfsIntegrationTest extends
   public AzureBlobFileSystem createFileSystem() throws IOException {
     Preconditions.checkState(abfs == null,
         "existing ABFS instance exists: %s", abfs);
-    abfs = (AzureBlobFileSystem) FileSystem.newInstance(configuration);
+    abfs = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig);
     return abfs;
   }
 
@@ -221,7 +222,7 @@ public abstract class AbstractAbfsIntegrationTest extends
 
   protected String getHostName() {
     // READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC
-    String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
+    String endPoint = abfsConfig.get(AZURE_ABFS_ENDPOINT);
     return endPoint.split(":")[0];
   }
 
@@ -245,13 +246,15 @@ public abstract class AbstractAbfsIntegrationTest extends
   }
 
   protected String getAccountKey() {
-    return configuration.get(
-        FS_AZURE_ACCOUNT_KEY_PREFIX
-            + accountName);
+    return abfsConfig.get(FS_AZURE_ACCOUNT_KEY);
+  }
+
+  public AbfsConfiguration getConfiguration() {
+    return abfsConfig;
   }
 
-  protected Configuration getConfiguration() {
-    return configuration;
+  public Configuration getRawConfiguration() {
+    return abfsConfig.getRawConfiguration();
   }
 
   protected boolean isIPAddress() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java
index cfda7a7..14c9bff 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsScaleTest.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.azurebfs;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
 
 import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled;
@@ -34,6 +35,10 @@ public class AbstractAbfsScaleTest extends AbstractAbfsIntegrationTest  {
   protected static final Logger LOG =
       LoggerFactory.getLogger(AbstractAbfsScaleTest.class);
 
+  public AbstractAbfsScaleTest() throws Exception {
+    super();
+  }
+
   @Override
   protected int getTestTimeoutMillis() {
     return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS;
@@ -43,7 +48,8 @@ public class AbstractAbfsScaleTest extends AbstractAbfsIntegrationTest  {
   public void setup() throws Exception {
     super.setup();
     LOG.debug("Scale test operation count = {}", getOperationCount());
-    assumeScaleTestsEnabled(getConfiguration());
+    Configuration rawConfiguration = getRawConfiguration();
+    assumeScaleTestsEnabled(rawConfiguration);
   }
 
   protected long getOperationCount() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
index 1c2083d..f024f25 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
@@ -30,6 +30,10 @@ import org.junit.Test;
 public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
   private static final int LIST_MAX_RESULTS = 5000;
 
+  public ITestAbfsClient() throws Exception {
+    super();
+  }
+
   @Test
   public void testContinuationTokenHavingEqualSign() throws Exception {
     final AzureBlobFileSystem fs = this.getFileSystem();
@@ -42,4 +46,4 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
       Assert.assertEquals("InvalidQueryParameterValue", ex.getErrorCode().getErrorCode());
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
index f62ea6e..a270a00 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
@@ -51,7 +51,7 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest {
 
   private final int size;
 
-  public ITestAbfsReadWriteAndSeek(final int size) {
+  public ITestAbfsReadWriteAndSeek(final int size) throws Exception {
     this.size = size;
   }
 
@@ -62,7 +62,7 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest {
 
   private void testReadWriteAndSeek(int bufferSize) throws Exception {
     final AzureBlobFileSystem fs = getFileSystem();
-    final AbfsConfiguration abfsConfiguration = new AbfsConfiguration(getConfiguration());
+    final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
 
     abfsConfiguration.setWriteBufferSize(bufferSize);
     abfsConfiguration.setReadBufferSize(bufferSize);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
index f2e26ec..cbe1939 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java
@@ -34,7 +34,8 @@ public class ITestAzureBlobFileSystemAppend extends
     AbstractAbfsIntegrationTest {
   private static final Path TEST_FILE_PATH = new Path("testfile");
   private static final Path TEST_FOLDER_PATH = new Path("testFolder");
-  public ITestAzureBlobFileSystemAppend() {
+
+  public ITestAzureBlobFileSystemAppend() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
index 46ac866..d8940f7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
@@ -35,7 +35,8 @@ import org.apache.hadoop.fs.Path;
  */
 public class ITestAzureBlobFileSystemBackCompat extends
     AbstractAbfsIntegrationTest {
-  public ITestAzureBlobFileSystemBackCompat() {
+
+  public ITestAzureBlobFileSystemBackCompat() throws Exception {
     super();
     Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
index 90eff97..917ee9c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
@@ -39,7 +39,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
  * Test copy operation.
  */
 public class ITestAzureBlobFileSystemCopy extends AbstractAbfsIntegrationTest {
-  public ITestAzureBlobFileSystemCopy() {
+
+  public ITestAzureBlobFileSystemCopy() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
index 1e43f9a..ab01166 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -38,7 +38,8 @@ public class ITestAzureBlobFileSystemCreate extends
   private static final Path TEST_FILE_PATH = new Path("testfile");
   private static final Path TEST_FOLDER_PATH = new Path("testFolder");
   private static final String TEST_CHILD_FILE = "childFile";
-  public ITestAzureBlobFileSystemCreate() {
+
+  public ITestAzureBlobFileSystemCreate() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
index 91d1723..486daca 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
@@ -41,7 +41,8 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
  */
 public class ITestAzureBlobFileSystemDelete extends
     AbstractAbfsIntegrationTest {
-  public ITestAzureBlobFileSystemDelete() {
+
+  public ITestAzureBlobFileSystemDelete() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
index f1800c0..6021e5d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
 
 import static org.junit.Assert.assertEquals;
@@ -45,9 +44,9 @@ public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest {
   private static final int TEST_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
   private static final int TEST_DEFAULT_READ_BUFFER_SIZE = 1023900;
 
-  public ITestAzureBlobFileSystemE2E() {
+  public ITestAzureBlobFileSystemE2E() throws Exception {
     super();
-    Configuration configuration = this.getConfiguration();
+    AbfsConfiguration configuration = this.getConfiguration();
     configuration.set(ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH, "0");
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
index 7ed9d42..fccd063 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
@@ -45,7 +45,7 @@ public class ITestAzureBlobFileSystemE2EScale extends
   private static final int ONE_MB = 1024 * 1024;
   private static final int DEFAULT_WRITE_TIMES = 100;
 
-  public ITestAzureBlobFileSystemE2EScale() {
+  public ITestAzureBlobFileSystemE2EScale() throws Exception {
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
index dba10f5..b08b920 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
@@ -41,7 +41,7 @@ public class ITestAzureBlobFileSystemFileStatus extends
   private static final Path TEST_FILE = new Path("testFile");
   private static final Path TEST_FOLDER = new Path("testDir");
 
-  public ITestAzureBlobFileSystemFileStatus() {
+  public ITestAzureBlobFileSystemFileStatus() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
index c1022b0..9d17388 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
@@ -41,12 +41,12 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
   @Test
   public void testFinalize() throws Exception {
     // Disable the cache for filesystem to make sure there is no reference.
-    Configuration configuration = this.getConfiguration();
-    configuration.setBoolean(
+    Configuration rawConfig = this.getRawConfiguration();
+    rawConfig.setBoolean(
             this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY,
     true);
 
-    AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(configuration);
+    AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig);
 
     WeakReference<Object> ref = new WeakReference<Object>(fs);
     fs = null;
@@ -61,4 +61,4 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
 
     Assert.assertTrue("testFinalizer didn't get cleaned up within maxTries", ref.get() == null);
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
index 337f95c..23a1ab5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
@@ -60,7 +60,7 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest {
   private static final int TEST_FILE_LENGTH = 1024 * 1024 * 8;
   private static final int WAITING_TIME = 1000;
 
-  public ITestAzureBlobFileSystemFlush() {
+  public ITestAzureBlobFileSystemFlush() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
index 874a8a3..5f08721 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
  */
 public class ITestAzureBlobFileSystemInitAndCreate extends
     AbstractAbfsIntegrationTest {
-  public ITestAzureBlobFileSystemInitAndCreate() {
 
+  public ITestAzureBlobFileSystemInitAndCreate() throws Exception {
     this.getConfiguration().unset(ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
index b87abe6..60e0fbc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
@@ -42,7 +42,8 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 public class ITestAzureBlobFileSystemListStatus extends
     AbstractAbfsIntegrationTest {
   private static final int TEST_FILES_NUMBER = 6000;
-  public ITestAzureBlobFileSystemListStatus() {
+
+  public ITestAzureBlobFileSystemListStatus() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
index 1bb2c54..382d396 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
@@ -28,7 +28,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
  * Test mkdir operation.
  */
 public class ITestAzureBlobFileSystemMkDir extends AbstractAbfsIntegrationTest {
-  public ITestAzureBlobFileSystemMkDir() {
+
+  public ITestAzureBlobFileSystemMkDir() throws Exception {
     super();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
index f60740f..533f471 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
@@ -53,7 +53,7 @@ public class ITestAzureBlobFileSystemOauth extends AbstractAbfsIntegrationTest{
   private static final Path EXISTED_FILE_PATH = new Path("/existedFile");
   private static final Path EXISTED_FOLDER_PATH = new Path("/existedFolder");
 
-  public ITestAzureBlobFileSystemOauth() {
+  public ITestAzureBlobFileSystemOauth() throws Exception {
     Assume.assumeTrue(this.getAuthType() == AuthType.OAuth);
   }
   /*
@@ -161,16 +161,18 @@ public class ITestAzureBlobFileSystemOauth extends AbstractAbfsIntegrationTest{
   }
 
   private AzureBlobFileSystem getBlobConributor() throws Exception {
-    Configuration configuration = this.getConfiguration();
-    configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID));
-    configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET));
-    return getFileSystem(configuration);
+    AbfsConfiguration abfsConfig = this.getConfiguration();
+    abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID));
+    abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET));
+    Configuration rawConfig = abfsConfig.getRawConfiguration();
+    return getFileSystem(rawConfig);
   }
 
   private AzureBlobFileSystem getBlobReader() throws Exception {
-    Configuration configuration = this.getConfiguration();
-    configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_READER_CLIENT_ID));
-    configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET));
-    return getFileSystem(configuration);
+    AbfsConfiguration abfsConfig = this.getConfiguration();
+    abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_READER_CLIENT_ID));
+    abfsConfig.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + "." + this.getAccountName(), abfsConfig.get(FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET));
+    Configuration rawConfig = abfsConfig.getRawConfiguration();
+    return getFileSystem(rawConfig);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
index c97e840..e0e1d89 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
@@ -41,7 +41,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
  */
 public class ITestAzureBlobFileSystemRename extends
     AbstractAbfsIntegrationTest {
-  public ITestAzureBlobFileSystemRename() {
+
+  public ITestAzureBlobFileSystemRename() throws Exception {
+    super();
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
index 0ac7fcf..044c325 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
@@ -66,7 +66,7 @@ public class ITestAzureBlobFileSystemRenameUnicode extends
         });
   }
 
-  public ITestAzureBlobFileSystemRenameUnicode() {
+  public ITestAzureBlobFileSystemRenameUnicode() throws Exception {
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
index 50b1828..8b60dd8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.azurebfs.services.AuthType;
  * Test AzureBlobFileSystem initialization.
  */
 public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest {
-  public ITestFileSystemInitialization() {
+  public ITestFileSystemInitialization() throws Exception {
     super();
   }
 
@@ -62,10 +62,10 @@ public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest {
         null,
         null,
         null);
-    Configuration conf = getConfiguration();
-    conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
+    Configuration rawConfig = getRawConfiguration();
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
 
-    try(SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.newInstance(conf)) {
+    try(SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.newInstance(rawConfig)) {
       assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME,
           filesystem + "@" + accountName,
           null,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
index 7a7e327..e6b45c8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path;
 public class ITestFileSystemProperties extends AbstractAbfsIntegrationTest {
   private static final int TEST_DATA = 100;
   private static final Path TEST_PATH = new Path("/testfile");
-  public ITestFileSystemProperties() {
+  public ITestFileSystemProperties() throws Exception {
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
index 56a91d3..4393bd8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
@@ -77,14 +77,15 @@ public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest {
 
   @Test
   public void ensureAzureBlobFileSystemIsDefaultFileSystem() throws Exception {
-    AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(getConfiguration());
+    Configuration rawConfig = getRawConfiguration();
+    AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig);
     assertNotNull("filesystem", fs);
 
     if (this.getAuthType() == AuthType.OAuth) {
-      Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem();
+      Abfss afs = (Abfss) FileContext.getFileContext(rawConfig).getDefaultFileSystem();
       assertNotNull("filecontext", afs);
     } else {
-      Abfs afs = (Abfs) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem();
+      Abfs afs = (Abfs) FileContext.getFileContext(rawConfig).getDefaultFileSystem();
       assertNotNull("filecontext", afs);
     }
 
@@ -100,13 +101,13 @@ public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest {
         null,
         null,
         null);
-    getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+    Configuration rawConfig = getRawConfiguration();
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
         defaultUri.toString());
 
-    SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.get(
-        getConfiguration());
+    SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.get(rawConfig);
     assertNotNull("filesystem", fs);
-    Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem();
+    Abfss afs = (Abfss) FileContext.getFileContext(rawConfig).getDefaultFileSystem();
     assertNotNull("filecontext", afs);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5593cbd/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
index bedd4f3..50f7bd9 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.azurebfs;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 
 import org.apache.commons.codec.Charsets;
@@ -66,6 +67,7 @@ public class TestAbfsConfigurationFieldsValidation {
   private static final int TEST_INT = 1234565;
   private static final int TEST_LONG = 4194304;
 
+  private final String accountName;
   private final String encodedString;
   private final String encodedAccountKey;
 
@@ -96,6 +98,7 @@ public class TestAbfsConfigurationFieldsValidation {
   public TestAbfsConfigurationFieldsValidation() throws Exception {
     super();
     Base64 base64 = new Base64();
+    this.accountName = "testaccount1.blob.core.windows.net";
     this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
     this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
     Configuration configuration = new Configuration();
@@ -105,8 +108,8 @@ public class TestAbfsConfigurationFieldsValidation {
     configuration.set(STRING_KEY, "stringValue");
     configuration.set(BASE64_KEY, encodedString);
     configuration.set(BOOLEAN_KEY, "true");
-    configuration.set(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "testaccount1.blob.core.windows.net", this.encodedAccountKey);
-    abfsConfiguration = new AbfsConfiguration(configuration);
+    configuration.set(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName, this.encodedAccountKey);
+    abfsConfiguration = new AbfsConfiguration(configuration, accountName);
   }
 
   @Test
@@ -143,30 +146,35 @@ public class TestAbfsConfigurationFieldsValidation {
 
   @Test
   public void testGetAccountKey() throws Exception {
-    String accountKey = abfsConfiguration.getStorageAccountKey("testaccount1.blob.core.windows.net");
+    String accountKey = abfsConfiguration.getStorageAccountKey();
     assertEquals(this.encodedAccountKey, accountKey);
   }
 
   @Test(expected = ConfigurationPropertyNotFoundException.class)
   public void testGetAccountKeyWithNonExistingAccountName() throws Exception {
-    abfsConfiguration.getStorageAccountKey("bogusAccountName");
+    Configuration configuration = new Configuration();
+    configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);
+    configuration.unset(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME);
+    AbfsConfiguration abfsConfig = new AbfsConfiguration(configuration, "bogusAccountName");
+    abfsConfig.getStorageAccountKey();
   }
 
   @Test
-  public void testSSLSocketFactoryConfiguration() throws InvalidConfigurationValueException, IllegalAccessException {
+  public void testSSLSocketFactoryConfiguration()
+      throws InvalidConfigurationValueException, IllegalAccessException, IOException {
     assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default, abfsConfiguration.getPreferredSSLFactoryOption());
     assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, abfsConfiguration.getPreferredSSLFactoryOption());
     assertNotEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, abfsConfiguration.getPreferredSSLFactoryOption());
 
     Configuration configuration = new Configuration();
     configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.Default_JSSE);
-    AbfsConfiguration localAbfsConfiguration = new AbfsConfiguration(configuration);
+    AbfsConfiguration localAbfsConfiguration = new AbfsConfiguration(configuration, accountName);
     assertEquals(SSLSocketFactoryEx.SSLChannelMode.Default_JSSE, localAbfsConfiguration.getPreferredSSLFactoryOption());
 
     configuration = new Configuration();
     configuration.setEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, SSLSocketFactoryEx.SSLChannelMode.OpenSSL);
-    localAbfsConfiguration = new AbfsConfiguration(configuration);
+    localAbfsConfiguration = new AbfsConfiguration(configuration, accountName);
     assertEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, localAbfsConfiguration.getPreferredSSLFactoryOption());
   }
 
-}
\ No newline at end of file
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message