From common-commits-return-88449-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Sun Sep 23 05:24:11 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 2C35E180677 for ; Sun, 23 Sep 2018 05:24:09 +0200 (CEST) Received: (qmail 795 invoked by uid 500); 23 Sep 2018 03:24:04 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 96422 invoked by uid 99); 23 Sep 2018 03:24:01 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 23 Sep 2018 03:24:01 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9B55BE120E; Sun, 23 Sep 2018 03:23:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mackrorysd@apache.org To: common-commits@hadoop.apache.org Date: Sun, 23 Sep 2018 03:24:26 -0000 Message-Id: In-Reply-To: <1537a6731926494e80f1b0e7a58d1509@git.apache.org> References: <1537a6731926494e80f1b0e7a58d1509@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [29/45] hadoop git commit: HADOOP-15663. ABFS: Simplify configuration. Contributed by Da Zhou. HADOOP-15663. ABFS: Simplify configuration. Contributed by Da Zhou. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81dc4a99 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81dc4a99 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81dc4a99 Branch: refs/heads/trunk Commit: 81dc4a995c3837b721a0c1f897698b5ed47b8fb0 Parents: df57c6c Author: Thomas Marquardt Authored: Fri Aug 31 03:24:42 2018 +0000 Committer: Thomas Marquardt Committed: Mon Sep 17 19:54:01 2018 +0000 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 12 ++ .../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 - .../fs/azurebfs/AzureBlobFileSystemStore.java | 24 ++- .../azurebfs/constants/ConfigurationKeys.java | 5 +- .../hadoop/fs/azurebfs/utils/UriUtils.java | 15 +- .../src/site/markdown/testing_azure.md | 209 ++++++++++++++----- .../fs/azure/AzureBlobStorageTestAccount.java | 22 +- ...TestFileSystemOperationExceptionMessage.java | 3 +- .../fs/azure/ITestWasbUriAndConfiguration.java | 26 --- .../azure/integration/AzureTestConstants.java | 6 +- .../fs/azure/integration/AzureTestUtils.java | 18 ++ .../azure/metrics/TestRollingWindowAverage.java | 4 +- .../azurebfs/AbstractAbfsIntegrationTest.java | 73 ++++--- .../azurebfs/AbstractAbfsTestWithTimeout.java | 70 +++++++ .../ITestAzureBlobFileSystemBackCompat.java | 2 +- .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2 +- .../TestAbfsConfigurationFieldsValidation.java | 3 +- .../constants/TestConfigurationKeys.java | 13 +- .../contract/ABFSContractTestBinding.java | 3 + .../TestConfigurationValidators.java | 1 - .../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 80 +++++++ .../utils/CleanUpAbfsTestContainer.java | 77 ------- .../hadoop/fs/azurebfs/utils/TestUriUtils.java | 12 +- .../src/test/resources/azure-bfs-test.xml | 188 ----------------- .../src/test/resources/azure-test.xml | 56 ++--- 25 files changed, 473 insertions(+), 455 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 471dacc..3fcdecb 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1619,6 +1619,18 @@ + fs.AbstractFileSystem.wasb.impl + org.apache.hadoop.fs.azure.Wasb + AbstractFileSystem implementation class of wasb:// + + + + fs.AbstractFileSystem.wasbs.impl + org.apache.hadoop.fs.azure.Wasbs + AbstractFileSystem implementation class of wasbs:// + + + fs.wasb.impl org.apache.hadoop.fs.azure.NativeAzureFileSystem The implementation class of the Native Azure Filesystem http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 069f17a..924bc3e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -173,10 +173,6 @@ public class AbfsConfiguration{ } } - public boolean isEmulator() { - return this.getConfiguration().getBoolean(FS_AZURE_EMULATOR_ENABLED, false); - } - public boolean isSecureMode() { return isSecure; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index fc60127..6542a64 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -77,6 +77,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -86,6 +87,7 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT; import static org.apache.hadoop.util.Time.now; /** @@ -146,7 +148,27 @@ public class AzureBlobFileSystemStore { final URIBuilder uriBuilder = new URIBuilder(); uriBuilder.setScheme(scheme); - uriBuilder.setHost(hostName); + + // For testing purposes, an IP address and port may be provided to override + // the host specified in the FileSystem URI. Also note that the format of + // the Azure Storage Service URI changes from + // http[s]://[account][domain-suffix]/[filesystem] to + // http[s]://[ip]:[port]/[account]/[filesystem]. + String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT); + if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) { + uriBuilder.setHost(hostName); + return uriBuilder; + } + + // Split ip and port + String[] data = endPoint.split(AbfsHttpConstants.COLON); + if (data.length != 2) { + throw new RuntimeException(String.format("ABFS endpoint is not set correctly : %s, " + + "Do not specify scheme when using {IP}:{PORT}", endPoint)); + } + uriBuilder.setHost(data[0].trim()); + uriBuilder.setPort(Integer.parseInt(data[1].trim())); + uriBuilder.setPath("/" + UriUtils.extractAccountNameFromHostName(hostName)); return uriBuilder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 8ad0dc6..ca4c9c3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -37,9 +37,6 @@ public final class ConfigurationKeys { public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval"; public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries"; - // Remove this and use common azure storage emulator property for public release. - public static final String FS_AZURE_EMULATOR_ENABLED = "fs.azure.abfs.emulator.enabled"; - // Read and write buffer sizes defined by the user public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size"; public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size"; @@ -60,6 +57,8 @@ public final class ConfigurationKeys { public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider."; public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; + /** End point of ABFS account: {@value}. */ + public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint"; /** Prefix for auth type properties: {@value}. */ public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type."; /** Prefix for oauth token provider type: {@value}. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java index 7652adf..1bbc1b3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java @@ -41,20 +41,21 @@ public final class UriUtils { } /** - * Extracts the raw account name from account name. - * @param accountName to extract the raw account name. - * @return extracted raw account name. + * Extracts the account name from the host name. + * @param hostName the fully-qualified domain name of the storage service + * endpoint (e.g. {account}.dfs.core.windows.net. + * @return the storage service account name. */ - public static String extractRawAccountFromAccountName(final String accountName) { - if (accountName == null || accountName.isEmpty()) { + public static String extractAccountNameFromHostName(final String hostName) { + if (hostName == null || hostName.isEmpty()) { return null; } - if (!containsAbfsUrl(accountName)) { + if (!containsAbfsUrl(hostName)) { return null; } - String[] splitByDot = accountName.split("\\."); + String[] splitByDot = hostName.split("\\."); if (splitByDot.length == 0) { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md index c148807..a4d920a 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md @@ -90,7 +90,7 @@ For example: - fs.azure.test.account.name + fs.azure.wasb.account.name {ACCOUNTNAME}.blob.core.windows.net @@ -126,7 +126,7 @@ Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml` - fs.azure.test.account.name + fs.azure.wasb.account.name {ACCOUNTNAME}.blob.core.windows.net @@ -576,77 +576,172 @@ This will delete the containers; the output log of the test run will provide the details and summary of the operation. -## Testing ABFS +## Testing the Azure ABFS Client -The ABFS Connector tests share the same account as the wasb tests; this is -needed for cross-connector compatibility tests. - -This makes for a somewhat complex set of configuration options. - -Here are the settings for an account `ACCOUNTNAME` +Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to +big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS +schemes target the ADLS Gen 2 REST API, and the WASB and WASBS schemes target +the Azure Blob Storage REST API. ADLS Gen 2 offers better performance and +scalability. ADLS Gen 2 also offers authentication and authorization compatible +with the Hadoop Distributed File System permissions model when hierarchical +namespace is enabled for the storage account. Furthermore, the metadata and data +produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa. + +In order to test ABFS, please add the following configuration to your +`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include +compatibility tests which require WASB credentials, in addition to the ABFS +credentials. ```xml - - abfs.account.name - ACCOUNTNAME - + + + + + fs.azure.abfs.account.name + {ACCOUNT_NAME}.dfs.core.windows.net + - - abfs.account.full.name - ${abfs.account.name}.dfs.core.windows.net - + + fs.azure.account.key.{ACCOUNT_NAME}.dfs.core.windows.net + {ACCOUNT_ACCESS_KEY} + - - abfs.account.key - SECRETKEY== - + + fs.azure.wasb.account.name + {ACCOUNT_NAME}.blob.core.windows.net + + + + fs.azure.account.key.{ACCOUNT_NAME}.blob.core.windows.net + {ACCOUNT_ACCESS_KEY} + - - fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net - ${abfs.account.key} - + + fs.contract.test.fs.abfs + abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net + A file system URI to be used by the contract tests. + - - fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net - ${abfs.account.key} - + + fs.contract.test.fs.wasb + wasb://{CONTAINER_NAME}@{ACCOUNT_NAME}.blob.core.windows.net + A file system URI to be used by the contract tests. + + +``` - - fs.azure.test.account.key.ACCOUNTNAME.dfs.core.windows.net - ${abfs.account.key} - +To run OAuth and ACL test cases you must use a storage account with the +hierarchical namespace enabled, and set the following configuration settings: +```xml + + - fs.azure.test.account.key.ACCOUNTNAME.blob.core.windows.net - ${abfs.account.key} + fs.azure.account.auth.type.{YOUR_ABFS_ACCOUNT_NAME} + {AUTH TYPE} + The authorization type can be SharedKey, OAuth, or Custom. The + default is SharedKey. - - fs.azure.account.key.ACCOUNTNAME - ${abfs.account.key} - + + + - - fs.azure.test.account.key.ACCOUNTNAME - ${abfs.account.key} - + - - fs.azure.test.account.name - ${abfs.account.full.name} - + - - fs.contract.test.fs.abfs - abfs://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net - Container for contract tests - + - - fs.contract.test.fs.abfss - abfss://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net - Container for contract tests - + + + + + + + + + + + ``` + +If running tests against an endpoint that uses the URL format +http[s]://[ip]:[port]/[account]/[filesystem] instead of +http[s]://[account][domain-suffix]/[filesystem], please use the following: + +```xml + + fs.azure.abfs.endpoint + {IP}:{PORT} + +``` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index 5b36c87..b65ce78 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig; /** * Helper class to create WASB file systems backed by either a mock in-memory @@ -58,11 +59,14 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, private static final Logger LOG = LoggerFactory.getLogger( AzureBlobStorageTestAccount.class); - private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; private static final String SAS_PROPERTY_NAME = "fs.azure.sas."; private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; - private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name"; + public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; + public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name"; + public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name"; public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net"; + public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net"; + public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net"; public static final String MOCK_CONTAINER_NAME = "mockContainer"; public static final String WASB_AUTHORITY_DELIMITER = "@"; public static final String WASB_SCHEME = "wasb"; @@ -379,7 +383,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, containerName); container.create(); - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); // Ensure that custom throttling is disabled and tolerate concurrent // out-of-band appends. @@ -525,7 +529,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, static CloudStorageAccount createTestAccount(Configuration conf) throws URISyntaxException, KeyProviderException { - String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String testAccountName = verifyWasbAccountNameInConfig(conf); if (testAccountName == null) { LOG.warn("Skipping live Azure test because of missing test account"); return null; @@ -570,16 +574,16 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, String containerName = useContainerSuffixAsContainerName ? containerNameSuffix : String.format( - "wasbtests-%s-%tQ%s", + "wasbtests-%s-%s%s", System.getProperty("user.name"), - new Date(), + UUID.randomUUID().toString(), containerNameSuffix); container = account.createCloudBlobClient().getContainerReference( containerName); if (createOptions.contains(CreateOptions.CreateContainer)) { container.createIfNotExists(); } - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); if (createOptions.contains(CreateOptions.UseSas)) { String sas = generateSAS(container, createOptions.contains(CreateOptions.Readonly)); @@ -741,7 +745,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, CloudBlobClient blobClient = account.createCloudBlobClient(); // Capture the account URL and the account name. - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); configureSecureModeTestSettings(conf); @@ -814,7 +818,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable, CloudBlobClient blobClient = account.createCloudBlobClient(); // Capture the account URL and the account name. - String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + String accountName = verifyWasbAccountNameInConfig(conf); configureSecureModeTestSettings(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java index 6d5e72e..af570bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java @@ -29,6 +29,7 @@ import com.microsoft.azure.storage.CloudStorageAccount; import org.junit.Test; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG; +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig; /** * Test for error messages coming from SDK. @@ -46,7 +47,7 @@ public class ITestFileSystemOperationExceptionMessage AzureBlobStorageTestAccount.createTestAccount(conf); AzureTestUtils.assume("No test account", account != null); - String testStorageAccount = conf.get("fs.azure.test.account.name"); + String testStorageAccount = verifyWasbAccountNameInConfig(conf); conf = new Configuration(); conf.set("fs.AbstractFileSystem.wasb.impl", "org.apache.hadoop.fs.azure.Wasb"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java index bee0220..7783684 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java @@ -500,32 +500,6 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout { } @Test - public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals("wasbs", fs.getScheme()); - - // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified - try{ - FileContext.getFileContext(conf).getDefaultFileSystem(); - fail("Should've thrown."); - }catch(UnsupportedFileSystemException e){ - } - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - @Test public void testCredentialProviderPathExclusions() throws Exception { String providerPath = "user:///,jceks://wasb/user/hrt_qa/sqoopdbpasswd.jceks," + http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java index 0b72f06..82907c5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java @@ -143,12 +143,10 @@ public interface AzureTestConstants { - String ACCOUNT_KEY_PROPERTY_NAME - = "fs.azure.account.key."; + String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key."; + String ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name"; String SAS_PROPERTY_NAME = "fs.azure.sas."; String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; - String TEST_ACCOUNT_NAME_PROPERTY_NAME - = "fs.azure.test.account.name"; String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net"; String MOCK_CONTAINER_NAME = "mockContainer"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java index 8d2a104..b438c8e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java @@ -36,10 +36,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; + +import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX; +import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN; import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; import static org.apache.hadoop.test.MetricsAsserts.getLongGauge; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.junit.Assume.assumeTrue; /** * Utilities for the Azure tests. Based on {@code S3ATestUtils}, so @@ -476,4 +480,18 @@ public final class AzureTestUtils extends Assert { + KEY_SCALE_TESTS_ENABLED, enabled); } + + /** + * Check the account name for WASB tests is set correctly and return. + */ + public static String verifyWasbAccountNameInConfig(Configuration conf) { + String accountName = conf.get(ACCOUNT_NAME_PROPERTY_NAME); + if (accountName == null) { + accountName = conf.get(WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN); + } + assumeTrue("Account for WASB is missing or it is not in correct format", + accountName != null && !accountName.endsWith(WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX)); + return accountName; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java index 9b1fb8e..cd8b692 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java @@ -18,8 +18,8 @@ package org.apache.hadoop.fs.azure.metrics; -import static org.junit.Assert.*; -import org.junit.*; +import static org.junit.Assert.assertEquals; +import org.junit.Test; public class TestRollingWindowAverage { /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index e0afeb4..f61954f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -35,7 +35,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; @@ -45,12 +44,12 @@ import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.io.IOUtils; +import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.assumeTrue; -import static org.hamcrest.CoreMatchers.*; /** * Base for AzureBlobFileSystem Integration tests. @@ -58,12 +57,12 @@ import static org.hamcrest.CoreMatchers.*; * Important: This is for integration tests only. */ public abstract class AbstractAbfsIntegrationTest extends - AbstractWasbTestWithTimeout { + AbstractAbfsTestWithTimeout { private static final Logger LOG = LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class); - private boolean isEmulator; + private boolean isIPAddress; private NativeAzureFileSystem wasb; private AzureBlobFileSystem abfs; private String abfsScheme; @@ -75,27 +74,28 @@ public abstract class AbstractAbfsIntegrationTest extends private AuthType authType; protected AbstractAbfsIntegrationTest() { - fileSystemName = ABFS_TEST_CONTAINER_PREFIX + UUID.randomUUID().toString(); + fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString(); configuration = new Configuration(); - configuration.addResource(ABFS_TEST_RESOURCE_XML); - this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME); + configuration.addResource(TEST_CONFIGURATION_FILE_NAME); + + this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME); + if (accountName == null) { + // check if accountName is set using different config key + accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME); + } + assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME, + accountName != null && !accountName.isEmpty()); + authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey); abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME : FileSystemUriSchemes.ABFS_SECURE_SCHEME; - String accountName = configuration.get(FS_AZURE_TEST_ACCOUNT_NAME, ""); - assumeTrue("Not set: " + FS_AZURE_TEST_ACCOUNT_NAME, - !accountName.isEmpty()); - assertThat("The key in " + FS_AZURE_TEST_ACCOUNT_KEY_PREFIX - + " is not bound to an ABFS account", - accountName, containsString("dfs.core.windows.net")); - String fullKey = FS_AZURE_TEST_ACCOUNT_KEY_PREFIX - + accountName; - if (authType == AuthType.SharedKey) { - assumeTrue("Not set: " + fullKey, configuration.get(fullKey) != null); + String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName; + assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null); + // Update credentials } else { String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName; assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null); @@ -113,7 +113,17 @@ public abstract class AbstractAbfsIntegrationTest extends this.testUrl = defaultUri.toString(); configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); - this.isEmulator = this.configuration.getBoolean(FS_AZURE_EMULATOR_ENABLED, false); + // For testing purposes, an IP address and port may be provided to override + // the host specified in the FileSystem URI. Also note that the format of + // the Azure Storage Service URI changes from + // http[s]://[account][domain-suffix]/[filesystem] to + // http[s]://[ip]:[port]/[account]/[filesystem]. + String endPoint = configuration.get(AZURE_ABFS_ENDPOINT); + if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) { + this.isIPAddress = true; + } else { + this.isIPAddress = false; + } } @@ -122,13 +132,22 @@ public abstract class AbstractAbfsIntegrationTest extends //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. createFileSystem(); - if (!isEmulator && authType == AuthType.SharedKey) { + if (!isIPAddress && authType == AuthType.SharedKey) { final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); final AzureNativeFileSystemStore azureNativeFileSystemStore = new AzureNativeFileSystemStore(); + + // update configuration with wasb credentials + String accountNameWithoutDomain = accountName.split("\\.")[0]; + String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX; + String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName; + if (configuration.get(keyProperty) == null) { + configuration.set(keyProperty, getAccountKey()); + } + azureNativeFileSystemStore.initialize( wasbUri, - getConfiguration(), + configuration, new AzureFileSystemInstrumentation(getConfiguration())); wasb = new NativeAzureFileSystem(azureNativeFileSystemStore); @@ -201,7 +220,9 @@ public abstract class AbstractAbfsIntegrationTest extends } protected String getHostName() { - return configuration.get(FS_AZURE_TEST_HOST_NAME); + // READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC + String endPoint = configuration.get(AZURE_ABFS_ENDPOINT); + return endPoint.split(":")[0]; } protected void setTestUrl(String testUrl) { @@ -220,21 +241,21 @@ public abstract class AbstractAbfsIntegrationTest extends } protected String getAccountName() { - return configuration.get(FS_AZURE_TEST_ACCOUNT_NAME); + return this.accountName; } protected String getAccountKey() { return configuration.get( - FS_AZURE_TEST_ACCOUNT_KEY_PREFIX - + getAccountName()); + FS_AZURE_ACCOUNT_KEY_PREFIX + + accountName); } protected Configuration getConfiguration() { return configuration; } - protected boolean isEmulator() { - return isEmulator; + protected boolean isIPAddress() { + return isIPAddress; } protected AuthType getAuthType() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java new file mode 100644 index 0000000..fee90ab --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; + +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT; + +/** + * Base class for any ABFS test with timeouts & named threads. + * This class does not attempt to bind to Azure. + */ +public class AbstractAbfsTestWithTimeout extends Assert { + /** + * The name of the current method. + */ + @Rule + public TestName methodName = new TestName(); + /** + * Set the timeout for every test. + * This is driven by the value returned by {@link #getTestTimeoutMillis()}. + */ + @Rule + public Timeout testTimeout = new Timeout(getTestTimeoutMillis()); + + /** + * Name the junit thread for the class. This will overridden + * before the individual test methods are run. + */ + @BeforeClass + public static void nameTestThread() { + Thread.currentThread().setName("JUnit"); + } + + /** + * Name the thread to the current test method. + */ + @Before + public void nameThread() { + Thread.currentThread().setName("JUnit-" + methodName.getMethodName()); + } + + /** + * Override point: the test timeout in milliseconds. + * @return a timeout in milliseconds + */ + protected int getTestTimeoutMillis() { + return TEST_TIMEOUT; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index 800b95a..46ac866 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -69,7 +69,7 @@ public class ITestAzureBlobFileSystemBackCompat extends private String getBlobConnectionString() { String connectionString; - if (isEmulator()) { + if (isIPAddress()) { connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://" + this.getHostName() + ":8880/" + this.getAccountName().split("\\.") [0] + ";AccountName=" + this.getAccountName().split("\\.")[0] http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index e4f3ea3..c4bfee2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -50,7 +50,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { LoggerFactory.getLogger(ITestWasbAbfsCompatibility.class); public ITestWasbAbfsCompatibility() throws Exception { - Assume.assumeFalse("Emulator is not supported", isEmulator()); + Assume.assumeFalse("Emulator is not supported", isIPAddress()); Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java index 556dcdb..bedd4f3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java @@ -23,6 +23,7 @@ import java.lang.reflect.Field; import org.apache.commons.codec.Charsets; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.BooleanConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation; @@ -98,7 +99,7 @@ public class TestAbfsConfigurationFieldsValidation { this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8); this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8); Configuration configuration = new Configuration(); - configuration.addResource("azure-bfs-test.xml"); + configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME); configuration.set(INT_KEY, "1234565"); configuration.set(LONG_KEY, "4194304"); configuration.set(STRING_KEY, "stringValue"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 67301c7..7852f25 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -22,10 +22,9 @@ package org.apache.hadoop.fs.azurebfs.constants; * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file. */ public final class TestConfigurationKeys { - public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name"; - public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key."; - public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name"; - public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port"; + public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name"; + public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name"; + public static final String FS_AZURE_ACCOUNT_KEY_PREFIX = "fs.azure.account.key."; public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id"; @@ -34,9 +33,9 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_ID = "fs.azure.account.oauth2.reader.client.id"; public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET = "fs.azure.account.oauth2.reader.client.secret"; - public static final String ABFS_TEST_RESOURCE_XML = "azure-bfs-test.xml"; - - public static final String ABFS_TEST_CONTAINER_PREFIX = "abfs-testcontainer-"; + public static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; + public static final String TEST_CONTAINER_PREFIX = "abfs-testcontainer-"; + public static final int TEST_TIMEOUT = 10 * 60 * 1000; private TestConfigurationKeys() {} } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java index 5505e6a..6d089d1 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.junit.Assume; /** * Bind ABFS contract tests to the Azure test setup/teardown. @@ -42,6 +43,8 @@ public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest { if (useExistingFileSystem) { Configuration configuration = getConfiguration(); String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI); + Assume.assumeTrue("Contract tests are skipped because of missing config property :" + + TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI, testUrl != null); if (getAuthType() != AuthType.SharedKey) { testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java index e9e90c9..a9556ab 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java @@ -32,7 +32,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.M import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE; -import static org.junit.Assert.assertEquals; /** * Test configuration validators. http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java new file mode 100644 index 0000000..9c636a2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.utils; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; + +import org.junit.Assume; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX; + + /* + * Some Utils for ABFS tests. + * */ +public final class AbfsTestUtils extends AbstractAbfsIntegrationTest{ + private static final Logger LOG = + LoggerFactory.getLogger(AbfsTestUtils.class); + /** + * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted. + * In that case, dev can use this tool to list and delete all test containers. + * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-" + */ + + public void checkContainers() throws Throwable { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + int count = 0; + CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); + CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); + Iterable containers + = blobClient.listContainers(TEST_CONTAINER_PREFIX); + for (CloudBlobContainer container : containers) { + count++; + LOG.info("Container {}, URI {}", + container.getName(), + container.getUri()); + } + LOG.info("Found {} test containers", count); + } + + + public void deleteContainers() throws Throwable { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + int count = 0; + CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); + CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); + Iterable containers + = blobClient.listContainers(TEST_CONTAINER_PREFIX); + for (CloudBlobContainer container : containers) { + LOG.info("Container {} URI {}", + container.getName(), + container.getUri()); + if (container.deleteIfExists()) { + count++; + } + } + LOG.info("Deleted {} test containers", count); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java deleted file mode 100644 index ef4ddb9..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.azurebfs.utils; - -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; - -import org.junit.Assume; -import org.junit.Test; - -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; -import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; -import org.apache.hadoop.fs.azurebfs.services.AuthType; - -import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS_TEST_CONTAINER_PREFIX; - -/** - * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted. - * In that case, dev can use this tool to list and delete all test containers. - * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-" - */ -public class CleanUpAbfsTestContainer extends AbstractAbfsIntegrationTest{ - - public CleanUpAbfsTestContainer() { - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); - } - - @Test - public void testEnumContainers() throws Throwable { - int count = 0; - CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); - CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); - Iterable containers - = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX); - for (CloudBlobContainer container : containers) { - count++; - System.out.println(String.format("Container %s URI %s", - container.getName(), - container.getUri())); - } - System.out.println(String.format("Found %d test containers", count)); - } - - @Test - public void testDeleteContainers() throws Throwable { - int count = 0; - CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount(); - CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); - Iterable containers - = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX); - for (CloudBlobContainer container : containers) { - System.out.println(String.format("Container %s URI %s", - container.getName(), - container.getUri())); - if (container.deleteIfExists()) { - count++; - } - } - System.out.println(String.format("Deleted %s test containers", count)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java index e71e31d..690e56c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -38,11 +38,11 @@ public final class TestUriUtils { @Test public void testExtractRawAccountName() throws Exception { - Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.core.windows.net")); - Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.preprod.core.windows.net")); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net")); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("")); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(null)); - Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net")); + Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.core.windows.net")); + Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.preprod.core.windows.net")); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net")); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("")); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(null)); + Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net")); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml deleted file mode 100644 index 9b908b0..0000000 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc4a99/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index 8cea256..a36a391 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -16,49 +16,39 @@ - + + + + fs.azure.test.emulator + false + - + + fs.azure.secure.mode + false + - - + + + fs.azure.user.agent.prefix MSFT - - - - + + + + - - - - fs.AbstractFileSystem.wasb.impl - org.apache.hadoop.fs.azure.Wasb - --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org