From common-commits-return-88009-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Sep 18 00:01:32 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 9F5A5180778 for ; Tue, 18 Sep 2018 00:01:29 +0200 (CEST) Received: (qmail 3385 invoked by uid 500); 17 Sep 2018 22:01:24 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 2516 invoked by uid 99); 17 Sep 2018 22:01:24 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Sep 2018 22:01:24 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DFD1BE11A4; Mon, 17 Sep 2018 22:01:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tmarquardt@apache.org To: common-commits@hadoop.apache.org Date: Mon, 17 Sep 2018 22:01:37 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/50] [abbrv] hadoop git commit: HADOOP-15660. ABFS: Add support for OAuth Contributed by Da Zhou, Rajeev Bansal, and Junhua Gu. http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index b1f1485..e0afeb4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; @@ -62,7 +63,7 @@ public abstract class AbstractAbfsIntegrationTest extends private static final Logger LOG = LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class); - private final boolean isEmulator; + private boolean isEmulator; private NativeAzureFileSystem wasb; private AzureBlobFileSystem abfs; private String abfsScheme; @@ -71,20 +72,18 @@ public abstract class AbstractAbfsIntegrationTest extends private String fileSystemName; private String accountName; private String testUrl; - - protected AbstractAbfsIntegrationTest(final boolean secure) { - this(secure ? FileSystemUriSchemes.ABFS_SECURE_SCHEME : FileSystemUriSchemes.ABFS_SCHEME); - } + private AuthType authType; protected AbstractAbfsIntegrationTest() { - this(FileSystemUriSchemes.ABFS_SCHEME); - } - - private AbstractAbfsIntegrationTest(final String scheme) { - abfsScheme = scheme; fileSystemName = ABFS_TEST_CONTAINER_PREFIX + UUID.randomUUID().toString(); configuration = new Configuration(); configuration.addResource(ABFS_TEST_RESOURCE_XML); + this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME); + + authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + + accountName, AuthType.SharedKey); + abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME + : FileSystemUriSchemes.ABFS_SECURE_SCHEME; String accountName = configuration.get(FS_AZURE_TEST_ACCOUNT_NAME, ""); assumeTrue("Not set: " + FS_AZURE_TEST_ACCOUNT_NAME, @@ -94,8 +93,13 @@ public abstract class AbstractAbfsIntegrationTest extends accountName, containsString("dfs.core.windows.net")); String fullKey = FS_AZURE_TEST_ACCOUNT_KEY_PREFIX + accountName; - assumeTrue("Not set: " + fullKey, - configuration.get(fullKey) != null); + + if (authType == AuthType.SharedKey) { + assumeTrue("Not set: " + fullKey, configuration.get(fullKey) != null); + } else { + String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName; + assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null); + } final String abfsUrl = this.getFileSystemName() + "@" + this.getAccountName(); URI defaultUri = null; @@ -110,7 +114,6 @@ public abstract class AbstractAbfsIntegrationTest extends configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); this.isEmulator = this.configuration.getBoolean(FS_AZURE_EMULATOR_ENABLED, false); - this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME); } @@ -119,7 +122,7 @@ public abstract class AbstractAbfsIntegrationTest extends //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. createFileSystem(); - if (!isEmulator) { + if (!isEmulator && authType == AuthType.SharedKey) { final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); final AzureNativeFileSystemStore azureNativeFileSystemStore = new AzureNativeFileSystemStore(); @@ -234,6 +237,10 @@ public abstract class AbstractAbfsIntegrationTest extends return isEmulator; } + protected AuthType getAuthType() { + return this.authType; + } + /** * Write a buffer to a file. * @param path path http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java index 9c369bb..1c2083d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.azurebfs; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; import org.junit.Assert; @@ -30,6 +29,7 @@ import org.junit.Test; */ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest { private static final int LIST_MAX_RESULTS = 5000; + @Test public void testContinuationTokenHavingEqualSign() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index d696481..6207a47 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -22,8 +22,11 @@ import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; + +import org.junit.Assume; import org.junit.Test; +import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -34,6 +37,7 @@ public class ITestAzureBlobFileSystemBackCompat extends AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemBackCompat() { super(); + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 791694b..13abaf8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -34,9 +34,6 @@ public class ITestAzureBlobFileSystemFileStatus extends AbstractAbfsIntegrationTest { private static final Path TEST_FILE = new Path("testFile"); private static final Path TEST_FOLDER = new Path("testDir"); - public ITestAzureBlobFileSystemFileStatus() { - super(); - } @Test public void testEnsureStatusWorksForRoot() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java index e4acbae..c1022b0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java @@ -25,12 +25,14 @@ import org.junit.Test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.services.AuthType; /** * Test finalize() method when "fs.abfs.impl.disable.cache" is enabled. */ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ - static final String DISABLE_CACHE_KEY = "fs.abfs.impl.disable.cache"; + static final String DISABLE_ABFS_CACHE_KEY = "fs.abfs.impl.disable.cache"; + static final String DISABLE_ABFSSS_CACHE_KEY = "fs.abfss.impl.disable.cache"; public ITestAzureBlobFileSystemFinalize() throws Exception { super(); @@ -40,7 +42,9 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ public void testFinalize() throws Exception { // Disable the cache for filesystem to make sure there is no reference. Configuration configuration = this.getConfiguration(); - configuration.setBoolean(this.DISABLE_CACHE_KEY, true); + configuration.setBoolean( + this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY, + true); AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(configuration); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 2f40b64..b02d723 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -31,9 +31,9 @@ import java.io.IOException; import com.microsoft.azure.storage.blob.BlockEntry; import com.microsoft.azure.storage.blob.BlockListingFilter; import com.microsoft.azure.storage.blob.CloudBlockBlob; -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; import org.hamcrest.core.IsEqual; import org.hamcrest.core.IsNot; +import org.junit.Assume; import org.junit.Test; import org.apache.hadoop.fs.FSDataInputStream; @@ -42,6 +42,9 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + /** * Test flush operation. */ @@ -209,6 +212,8 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest { @Test public void testFlushWithFlushEnabled() throws Exception { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + AzureBlobStorageTestAccount testAccount = createWasbTestAccount(); String wasbUrl = testAccount.getFileSystem().getName(); String abfsUrl = wasbUrlToAbfsUrl(wasbUrl); @@ -228,6 +233,7 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest { @Test public void testFlushWithFlushDisabled() throws Exception { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); AzureBlobStorageTestAccount testAccount = createWasbTestAccount(); String wasbUrl = testAccount.getFileSystem().getName(); String abfsUrl = wasbUrlToAbfsUrl(wasbUrl); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java new file mode 100644 index 0000000..f60740f --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; +import org.junit.Assume; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_DATA_READER_CLIENT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET; + +/** + * Test Azure Oauth with Blob Data contributor role and Blob Data Reader role. + * The Test AAD client need to be configured manually through Azure Portal, then save their properties in + * configuration files. + */ +public class ITestAzureBlobFileSystemOauth extends AbstractAbfsIntegrationTest{ + + private static final Path FILE_PATH = new Path("/testFile"); + private static final Path EXISTED_FILE_PATH = new Path("/existedFile"); + private static final Path EXISTED_FOLDER_PATH = new Path("/existedFolder"); + + public ITestAzureBlobFileSystemOauth() { + Assume.assumeTrue(this.getAuthType() == AuthType.OAuth); + } + /* + * BLOB DATA CONTRIBUTOR should have full access to the container and blobs in the container. + * */ + @Test + public void testBlobDataContributor() throws Exception { + String clientId = this.getConfiguration().get(TestConfigurationKeys.FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID); + Assume.assumeTrue("Contributor client id not provided", clientId != null); + String secret = this.getConfiguration().get(TestConfigurationKeys.FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET); + Assume.assumeTrue("Contributor client secret not provided", secret != null); + + prepareFiles(); + + final AzureBlobFileSystem fs = getBlobConributor(); + + // create and write into file in current container/fs + try(FSDataOutputStream stream = fs.create(FILE_PATH)) { + stream.write(0); + } + assertTrue(fs.exists(FILE_PATH)); + FileStatus fileStatus = fs.getFileStatus(FILE_PATH); + assertEquals(1, fileStatus.getLen()); + // delete file + assertTrue(fs.delete(FILE_PATH, true)); + assertFalse(fs.exists(FILE_PATH)); + + // Verify Blob Data Contributor has full access to existed folder, file + + // READ FOLDER + assertTrue(fs.exists(EXISTED_FOLDER_PATH)); + + //DELETE FOLDER + fs.delete(EXISTED_FOLDER_PATH, true); + assertFalse(fs.exists(EXISTED_FOLDER_PATH)); + + // READ FILE + try (FSDataInputStream stream = fs.open(EXISTED_FILE_PATH)) { + assertTrue(stream.read() != 0); + } + + assertEquals(0, fs.getFileStatus(EXISTED_FILE_PATH).getLen()); + + // WRITE FILE + try (FSDataOutputStream stream = fs.append(EXISTED_FILE_PATH)) { + stream.write(0); + } + + assertEquals(1, fs.getFileStatus(EXISTED_FILE_PATH).getLen()); + + // REMOVE FILE + fs.delete(EXISTED_FILE_PATH, true); + assertFalse(fs.exists(EXISTED_FILE_PATH)); + } + + /* + * BLOB DATA READER should have only READ access to the container and blobs in the container. + * */ + @Test + public void testBlobDataReader() throws Exception { + String clientId = this.getConfiguration().get(TestConfigurationKeys.FS_AZURE_BLOB_DATA_READER_CLIENT_ID); + Assume.assumeTrue("Reader client id not provided", clientId != null); + String secret = this.getConfiguration().get(TestConfigurationKeys.FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET); + Assume.assumeTrue("Reader client secret not provided", secret != null); + + prepareFiles(); + final AzureBlobFileSystem fs = getBlobReader(); + + // Use abfsStore in this test to verify the ERROR code in AbfsRestOperationException + AzureBlobFileSystemStore abfsStore = fs.getAbfsStore(); + // TEST READ FS + Map properties = abfsStore.getFilesystemProperties(); + // TEST READ FOLDER + assertTrue(fs.exists(EXISTED_FOLDER_PATH)); + + // TEST DELETE FOLDER + try { + abfsStore.delete(EXISTED_FOLDER_PATH, true); + } catch (AbfsRestOperationException e) { + assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode()); + } + + // TEST READ FILE + try (InputStream inputStream = abfsStore.openFileForRead(EXISTED_FILE_PATH, null)) { + assertTrue(inputStream.read() != 0); + } + + // TEST WRITE FILE + try { + abfsStore.openFileForWrite(EXISTED_FILE_PATH, true); + } catch (AbfsRestOperationException e) { + assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode()); + } + + } + + private void prepareFiles() throws IOException { + // create test files/folders to verify access control diff between + // Blob data contributor and Blob data reader + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.create(EXISTED_FILE_PATH); + assertTrue(fs.exists(EXISTED_FILE_PATH)); + fs.mkdirs(EXISTED_FOLDER_PATH); + assertTrue(fs.exists(EXISTED_FOLDER_PATH)); + } + + private AzureBlobFileSystem getBlobConributor() throws Exception { + Configuration configuration = this.getConfiguration(); + configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID)); + configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET)); + return getFileSystem(configuration); + } + + private AzureBlobFileSystem getBlobReader() throws Exception { + Configuration configuration = this.getConfiguration(); + configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_READER_CLIENT_ID)); + configuration.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + this.getAccountName(), configuration.get(FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET)); + return getFileSystem(configuration); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java index c61de67..13c5bc8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Random; import java.util.concurrent.Callable; +import org.junit.Assume; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +35,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; +import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.contract.ContractTestUtils; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -66,6 +68,7 @@ public class ITestAzureBlobFileSystemRandomRead extends public ITestAzureBlobFileSystemRandomRead() throws Exception { super(); + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java index 3a44909..50b1828 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.services.AuthType; /** * Test AzureBlobFileSystem initialization. @@ -41,8 +42,10 @@ public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest { final String accountName = getAccountName(); final String filesystem = getFileSystemName(); + String scheme = this.getAuthType() == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME + : FileSystemUriSchemes.ABFS_SECURE_SCHEME; assertEquals(fs.getUri(), - new URI(FileSystemUriSchemes.ABFS_SCHEME, + new URI(scheme, filesystem + "@" + accountName, null, null, http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java index 5d1cf91..56a91d3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.services.AuthType; /** * Test AzureBlobFileSystem registration. @@ -79,8 +80,14 @@ public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest { AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(getConfiguration()); assertNotNull("filesystem", fs); - Abfs afs = (Abfs) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); - assertNotNull("filecontext", afs); + if (this.getAuthType() == AuthType.OAuth) { + Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + assertNotNull("filecontext", afs); + } else { + Abfs afs = (Abfs) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + assertNotNull("filecontext", afs); + } + } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index a89c044..ff28d3e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.azurebfs.services.AuthType; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory; @@ -50,6 +51,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { public ITestWasbAbfsCompatibility() throws Exception { Assume.assumeFalse("Emulator is not supported", isEmulator()); + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index fc7312a..67301c7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -28,6 +28,12 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port"; public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; + public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id"; + public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET = "fs.azure.account.oauth2.contributor.client.secret"; + + public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_ID = "fs.azure.account.oauth2.reader.client.id"; + public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET = "fs.azure.account.oauth2.reader.client.secret"; + public static final String ABFS_TEST_RESOURCE_XML = "azure-bfs-test.xml"; public static final String ABFS_TEST_CONTAINER_PREFIX = "abfs-testcontainer-"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java index ffd5bab..5505e6a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.services.AuthType; /** * Bind ABFS contract tests to the Azure test setup/teardown. @@ -32,18 +33,17 @@ import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest { private final URI testUri; - public ABFSContractTestBinding(final boolean secure) throws Exception { - this(secure, true); + public ABFSContractTestBinding() throws Exception { + this(true); } - public ABFSContractTestBinding(final boolean secure, + public ABFSContractTestBinding( final boolean useExistingFileSystem) throws Exception{ - super(secure); if (useExistingFileSystem) { Configuration configuration = getConfiguration(); String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI); - if (secure) { + if (getAuthType() != AuthType.SharedKey) { testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME); } setTestUrl(testUrl); @@ -61,4 +61,8 @@ public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest { public Configuration getConfiguration() { return super.getConfiguration(); } + + public boolean isSecureMode() { + return this.getAuthType() == AuthType.SharedKey ? false : true; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java index a302fcc..8a955bc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java @@ -18,34 +18,23 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractAppendTest; import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Test; import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; /** * Contract test for open operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractAppend extends AbstractContractAppendTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractAppend(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(this.isSecure); + public ITestAbfsFileSystemContractAppend() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java index c31a6d2..383528b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java @@ -17,11 +17,6 @@ */ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractConcatTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -29,19 +24,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for concat operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractConcat extends AbstractContractConcatTest{ - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractConcat(final boolean secure) throws Exception { - isSecure = secure; - binding = new ABFSContractTestBinding(isSecure); + public ITestAbfsFileSystemContractConcat() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java index ce4d229..3c3e949 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractCreateTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -30,19 +25,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for create operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractCreate extends AbstractContractCreateTest{ - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractCreate(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(this.isSecure); + public ITestAbfsFileSystemContractCreate() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java index 310731c..1d1136c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -30,19 +25,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for delete operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractDelete extends AbstractContractDeleteTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractDelete(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(isSecure); + public ITestAbfsFileSystemContractDelete() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java index c2cf255..544bbbf 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java @@ -28,7 +28,7 @@ public class ITestAbfsFileSystemContractDistCp extends AbstractContractDistCpTes private final ABFSContractTestBinding binding; public ITestAbfsFileSystemContractDistCp() throws Exception { - binding = new ABFSContractTestBinding(false); + binding = new ABFSContractTestBinding(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java index 9ad3b21..08b7eef 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java @@ -17,11 +17,6 @@ */ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -29,19 +24,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for getFileStatus operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractGetFileStatus extends AbstractContractGetFileStatusTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractGetFileStatus(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(isSecure); + public ITestAbfsFileSystemContractGetFileStatus() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java index 6265ca1..7b78575 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -30,19 +25,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for mkdir operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractMkdir extends AbstractContractMkdirTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractMkdir(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(secure); + public ITestAbfsFileSystemContractMkdir() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java index ae4bb2a..41f691d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractOpenTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -30,19 +25,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for open operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractOpen extends AbstractContractOpenTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractOpen(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(this.isSecure); + public ITestAbfsFileSystemContractOpen() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java index 6e6a728..82f104a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractRenameTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -30,19 +25,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for rename operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractRename extends AbstractContractRenameTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractRename(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(this.isSecure); + public ITestAbfsFileSystemContractRename() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java index 01dea2d..5b5493f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java @@ -17,31 +17,21 @@ */ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.junit.Ignore; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; /** * Contract test for root directory operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractRootDirectory extends AbstractContractRootDirectoryTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractRootDirectory(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(secure); + public ITestAbfsFileSystemContractRootDirectory() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java index 5ed7466..fc235e3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java @@ -28,7 +28,7 @@ public class ITestAbfsFileSystemContractSecureDistCp extends AbstractContractDis private final ABFSContractTestBinding binding; public ITestAbfsFileSystemContractSecureDistCp() throws Exception { - binding = new ABFSContractTestBinding(true); + binding = new ABFSContractTestBinding(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java index 5e0ea0c..4529e75 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java @@ -18,11 +18,6 @@ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractSeekTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -30,19 +25,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for seek operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractSeek extends AbstractContractSeekTest{ - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractSeek(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(this.isSecure); + public ITestAbfsFileSystemContractSeek() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java index 8d23b0b..6c4f9ba 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java @@ -17,11 +17,6 @@ */ package org.apache.hadoop.fs.azurebfs.contract; -import java.util.Arrays; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractSetTimesTest; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -29,19 +24,13 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Contract test for setTimes operation. */ -@RunWith(Parameterized.class) public class ITestAbfsFileSystemContractSetTimes extends AbstractContractSetTimesTest { - @Parameterized.Parameters(name = "SecureMode={0}") - public static Iterable secure() { - return Arrays.asList(new Object[][] { {true}, {false} }); - } - private final boolean isSecure; private final ABFSContractTestBinding binding; - public ITestAbfsFileSystemContractSetTimes(final boolean secure) throws Exception { - this.isSecure = secure; - binding = new ABFSContractTestBinding(this.isSecure); + public ITestAbfsFileSystemContractSetTimes() throws Exception { + binding = new ABFSContractTestBinding(); + this.isSecure = binding.isSecureMode(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java index d8854a2..a9fa2d7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java @@ -40,7 +40,7 @@ public class ITestAzureBlobFileSystemBasics extends FileSystemContractBaseTest { public ITestAzureBlobFileSystemBasics() throws Exception { // If all contract tests are running in parallel, some root level tests in FileSystemContractBaseTest will fail // due to the race condition. Hence for this contract test it should be tested in different container - binding = new ABFSContractTestBinding(false, false); + binding = new ABFSContractTestBinding(false); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 7bb27fc..a2fdd09 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -21,11 +21,11 @@ package org.apache.hadoop.fs.azurebfs.services; import java.net.URL; import java.util.regex.Pattern; -import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; -import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; import org.junit.Assert; import org.junit.Test; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.utils.SSLSocketFactoryEx; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; @@ -40,7 +40,7 @@ public final class TestAbfsClient { AbfsConfiguration config, boolean includeSSLProvider) { AbfsClient client = new AbfsClient(baseUrl, null, - config, null); + config, null, null); String sslProviderName = null; if (includeSSLProvider) { sslProviderName = SSLSocketFactoryEx.getDefaultFactory().getProviderName(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestQueryParams.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestQueryParams.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestQueryParams.java new file mode 100644 index 0000000..e6c6993 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestQueryParams.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.hadoop.fs.azurebfs.oauth2.QueryParams; +/** + * Test query params serialization. + */ +public class TestQueryParams { + private static final String SEPARATOR = "&"; + private static final String[][] PARAM_ARRAY = {{"K0", "V0"}, {"K1", "V1"}, {"K2", "V2"}}; + + @Test + public void testOneParam() { + String key = PARAM_ARRAY[0][0]; + String value = PARAM_ARRAY[0][1]; + + Map paramMap = new HashMap<>(); + paramMap.put(key, value); + + QueryParams qp = new QueryParams(); + qp.add(key, value); + Assert.assertEquals(key + "=" + value, qp.serialize()); + } + + @Test + public void testMultipleParams() { + QueryParams qp = new QueryParams(); + for (String[] entry : PARAM_ARRAY) { + qp.add(entry[0], entry[1]); + } + Map paramMap = constructMap(qp.serialize()); + Assert.assertEquals(PARAM_ARRAY.length, paramMap.size()); + + for (String[] entry : PARAM_ARRAY) { + Assert.assertTrue(paramMap.containsKey(entry[0])); + Assert.assertEquals(entry[1], paramMap.get(entry[0])); + } + } + + private Map constructMap(String input) { + String[] entries = input.split(SEPARATOR); + Map paramMap = new HashMap<>(); + for (String entry : entries) { + String[] keyValue = entry.split("="); + paramMap.put(keyValue[0], keyValue[1]); + } + return paramMap; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java index 9051a72..ef4ddb9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java @@ -21,9 +21,13 @@ import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.junit.Assume; import org.junit.Test; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS_TEST_CONTAINER_PREFIX; /** @@ -31,7 +35,12 @@ import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS * In that case, dev can use this tool to list and delete all test containers. * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-" */ -public class CleanUpAbfsTestContainer { +public class CleanUpAbfsTestContainer extends AbstractAbfsIntegrationTest{ + + public CleanUpAbfsTestContainer() { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + } + @Test public void testEnumContainers() throws Throwable { int count = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9149b970/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml index 464a8e6..9b908b0 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml @@ -23,6 +23,27 @@ + fs.azure.account.auth.type.{YOURACCOUNT}.dfs.core.windows.net + {AUTH TYPE} + The auth type can be : SharedKey, OAuth, Custom. By default "SharedKey" is used. + + + + fs.contract.test.fs.abfs + abfs://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value> + The name of the azure file system for testing. + + + + fs.contract.test.fs.abfss + abfss://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value> + The name of the azure file system for testing. + + --> + + + + + + + + + + + + + + + + + + + + + + + + --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org