hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [1/2] hadoop git commit: HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)
Date Sun, 08 Mar 2015 18:22:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 02cadbd24 -> b8a208795
  refs/heads/trunk 608ebd52b -> 64443490d


HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a20879
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a20879
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a20879

Branch: refs/heads/branch-2
Commit: b8a208795126932e683ce73bebcf99690540f9a8
Parents: 02cadbd
Author: Steve Loughran <stevel@apache.org>
Authored: Sun Mar 8 11:20:42 2015 -0700
Committer: Steve Loughran <stevel@apache.org>
Committed: Sun Mar 8 11:22:01 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  2 ++
 .../org/apache/hadoop/fs/s3a/Constants.java     |  6 +++++-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 20 ++++++++++++++------
 .../src/site/markdown/tools/hadoop-aws/index.md | 10 +++++-----
 4 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a20879/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9e4069e..d92d250 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -640,6 +640,8 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11674. oneByteBuf in CryptoInputStream and CryptoOutputStream
     should be non static. (Sean Busbey via yliu)
 
+    HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a20879/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index e7462dc..3486dfb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -18,8 +18,12 @@
 
 package org.apache.hadoop.fs.s3a;
 
-
 public class Constants {
+  // s3 access key
+  public static final String ACCESS_KEY = "fs.s3a.access.key";
+
+  // s3 secret key
+  public static final String SECRET_KEY = "fs.s3a.secret.key";
 
   // number of simultaneous connections to s3
   public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a20879/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 1a30d6f..91a606c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -32,8 +32,6 @@ import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.hadoop.fs.s3.S3Credentials;
-
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
@@ -159,12 +157,22 @@ public class S3AFileSystem extends FileSystem {
         this.getWorkingDirectory());
 
     // Try to get our credentials or just connect anonymously
-    S3Credentials s3Credentials = new S3Credentials();
-    s3Credentials.initialize(name, conf);
+    String accessKey = conf.get(ACCESS_KEY, null);
+    String secretKey = conf.get(SECRET_KEY, null);
+
+    String userInfo = name.getUserInfo();
+    if (userInfo != null) {
+      int index = userInfo.indexOf(':');
+      if (index != -1) {
+        accessKey = userInfo.substring(0, index);
+        secretKey = userInfo.substring(index + 1);
+      } else {
+        accessKey = userInfo;
+      }
+    }
 
     AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
-        new BasicAWSCredentialsProvider(s3Credentials.getAccessKey(),
-                                        s3Credentials.getSecretAccessKey()),
+        new BasicAWSCredentialsProvider(accessKey, secretKey),
         new InstanceProfileCredentialsProvider(),
         new AnonymousAWSCredentialsProvider()
     );

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a20879/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index bf62634..e0389c0 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -141,12 +141,12 @@ If you do any of these: change your credentials immediately!
 ### Authentication properties
 
     <property>
-      <name>fs.s3a.awsAccessKeyId</name>
+      <name>fs.s3a.access.key</name>
       <description>AWS access key ID. Omit for Role-based authentication.</description>
     </property>
 
     <property>
-      <name>fs.s3a.awsSecretAccessKey</name>
+      <name>fs.s3a.secret.key</name>
       <description>AWS secret key. Omit for Role-based authentication.</description>
     </property>
 
@@ -411,13 +411,13 @@ Example:
       </property>
 
       <property>
-        <name>fs.s3a.awsAccessKeyId</name>
+        <name>fs.s3a.access.key</name>
         <description>AWS access key ID. Omit for Role-based authentication.</description>
-        <value>DONOTPCOMMITTHISKEYTOSCM</value>
+        <value>DONOTCOMMITTHISKEYTOSCM</value>
       </property>
   
       <property>
-        <name>fs.s3a.awsSecretAccessKey</name>
+        <name>fs.s3a.secret.key</name>
         <description>AWS secret key. Omit for Role-based authentication.</description>
         <value>DONOTEVERSHARETHISSECRETKEY!</value>
       </property>


Mime
View raw message