Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 49C92200B45 for ; Fri, 10 Jun 2016 05:02:46 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 483C9160A5C; Fri, 10 Jun 2016 03:02:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 4703E160A5D for ; Fri, 10 Jun 2016 05:02:45 +0200 (CEST) Received: (qmail 56025 invoked by uid 500); 10 Jun 2016 03:02:40 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 53348 invoked by uid 99); 10 Jun 2016 03:02:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Jun 2016 03:02:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0495BE08BD; Fri, 10 Jun 2016 03:02:39 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Fri, 10 Jun 2016 03:02:54 -0000 Message-Id: <14dbc7f40b33422082031898bfce1a6e@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [17/24] hadoop git commit: HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth archived-at: Fri, 10 Jun 2016 03:02:46 -0000 HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/656c460c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/656c460c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/656c460c Branch: refs/heads/HDFS-1312 Commit: 656c460c0e79ee144d6ef48d85cec04a1af3b2cc Parents: 8ea9bbc Author: Steve Loughran Authored: Thu Jun 9 16:36:27 2016 +0100 Committer: Steve Loughran Committed: Thu Jun 9 17:28:49 2016 +0100 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 13 ++++- .../fs/s3a/AnonymousAWSCredentialsProvider.java | 11 ++++ .../fs/s3a/BasicAWSCredentialsProvider.java | 8 +++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 22 +++++--- .../src/site/markdown/tools/hadoop-aws/index.md | 14 ++++- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 55 ++++++++++++++++++++ 6 files changed, 113 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c460c/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index a65246b..8bb27ea 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -791,7 +791,18 @@ fs.s3a.aws.credentials.provider - Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys or another authentication mechanism. + + Class name of a credentials provider that implements + com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows + anonymous access to a publicly accessible S3 bucket without any credentials. + Please note that allowing anonymous access to an S3 bucket compromises + security and therefore is unsuitable for most use cases. It can be useful + for accessing public data sets without requiring AWS credentials. + http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c460c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index e62ec77..2c863fc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -24,6 +24,17 @@ import com.amazonaws.auth.AWSCredentials; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * AnonymousAWSCredentialsProvider supports anonymous access to AWS services + * through the AWS SDK. AWS requests will not be signed. This is not suitable + * for most cases, because allowing anonymous access to an S3 bucket compromises + * security. This can be useful for accessing public data sets without + * requiring AWS credentials. + * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + */ @InterfaceAudience.Private @InterfaceStability.Stable public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c460c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java index 2f721e4..3a5ee8c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java @@ -26,6 +26,14 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * BasicAWSCredentialsProvider supports static configuration of access key ID + * and secret access key for use with the AWS SDK. + * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + */ @InterfaceAudience.Private @InterfaceStability.Stable public class BasicAWSCredentialsProvider implements AWSCredentialsProvider { http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c460c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0281a3a..9af0a99 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -465,20 +465,28 @@ public class S3AFileSystem extends FileSystem { new BasicAWSCredentialsProvider( creds.getAccessKey(), creds.getAccessSecret()), new InstanceProfileCredentialsProvider(), - new EnvironmentVariableCredentialsProvider(), - new AnonymousAWSCredentialsProvider() - ); + new EnvironmentVariableCredentialsProvider()); } else { try { LOG.debug("Credential provider class is {}", className); - credentials = (AWSCredentialsProvider) Class.forName(className) - .getDeclaredConstructor(URI.class, Configuration.class) - .newInstance(this.uri, conf); + Class credClass = Class.forName(className); + try { + credentials = + (AWSCredentialsProvider)credClass.getDeclaredConstructor( + URI.class, Configuration.class).newInstance(this.uri, conf); + } catch (NoSuchMethodException | SecurityException e) { + credentials = + (AWSCredentialsProvider)credClass.getDeclaredConstructor() + .newInstance(); + } } catch (ClassNotFoundException e) { throw new IOException(className + " not found.", e); } catch (NoSuchMethodException | SecurityException e) { - throw new IOException(className + " constructor exception.", e); + throw new IOException(String.format("%s constructor exception. A " + + "class specified in %s must provide an accessible constructor " + + "accepting URI and Configuration, or an accessible default " + + "constructor.", className, AWS_CREDENTIALS_PROVIDER), e); } catch (ReflectiveOperationException | IllegalArgumentException e) { throw new IOException(className + " instantiation exception.", e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c460c/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 7d63a86..4086bc0 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -187,8 +187,18 @@ If you do any of these: change your credentials immediately! fs.s3a.aws.credentials.provider - Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. - Omit if using access/secret keys or another authentication mechanism. + + Class name of a credentials provider that implements + com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows + anonymous access to a publicly accessible S3 bucket without any credentials. + Please note that allowing anonymous access to an S3 bucket compromises + security and therefore is unsuitable for most use cases. It can be useful + for accessing public data sets without requiring AWS credentials. + #### Protecting the AWS Credentials in S3A http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c460c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 1a11a45..a25ca9c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; import static org.junit.Assert.*; import java.io.IOException; @@ -26,8 +27,13 @@ import java.net.URI; import java.nio.file.AccessDeniedException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; @@ -45,6 +51,12 @@ public class TestS3AAWSCredentialsProvider { private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class); + @Rule + public Timeout testTimeout = new Timeout(1 * 60 * 1000); + + @Rule + public ExpectedException exception = ExpectedException.none(); + @Test public void testBadConfiguration() throws IOException { Configuration conf = new Configuration(); @@ -113,4 +125,47 @@ public class TestS3AAWSCredentialsProvider { conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName()); S3ATestUtils.createTestFileSystem(conf); } + + @Test + public void testAnonymousProvider() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + AnonymousAWSCredentialsProvider.class.getName()); + Path testFile = new Path( + conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); + FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); + assertNotNull(fs); + assertTrue(fs instanceof S3AFileSystem); + FileStatus stat = fs.getFileStatus(testFile); + assertNotNull(stat); + assertEquals(testFile, stat.getPath()); + } + + static class ConstructorErrorProvider implements AWSCredentialsProvider { + + @SuppressWarnings("unused") + public ConstructorErrorProvider(String str) { + } + + @Override + public AWSCredentials getCredentials() { + return null; + } + + @Override + public void refresh() { + } + } + + @Test + public void testProviderConstructorError() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + ConstructorErrorProvider.class.getName()); + Path testFile = new Path( + conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); + exception.expect(IOException.class); + exception.expectMessage("constructor exception"); + FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org