Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: cnauroth@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <869c6aa4860440e89606e769f99cce20@git.apache.org>
Subject: hadoop git commit: HADOOP-14248. Retire
 SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang
 Liu.
Date: Wed, 12 Apr 2017 17:06:11 +0000 (UTC)
archived-at: Wed, 12 Apr 2017 17:06:13 -0000

Repository: hadoop
Updated Branches:
  refs/heads/trunk a16ab2be9 -> b8305e6d0


HADOOP-14248. Retire SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang Liu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8305e6d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8305e6d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8305e6d

Branch: refs/heads/trunk
Commit: b8305e6d06cbb9d44c728da14506d5cf825c12e5
Parents: a16ab2b
Author: Chris Nauroth <cnauroth@apache.org>
Authored: Wed Apr 12 10:02:13 2017 -0700
Committer: Chris Nauroth <cnauroth@apache.org>
Committed: Wed Apr 12 10:02:13 2017 -0700

----------------------------------------------------------------------
 .../src/main/resources/core-default.xml         |  9 +--
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  8 +--
 ...haredInstanceProfileCredentialsProvider.java | 67 --------------------
 .../src/site/markdown/tools/hadoop-aws/index.md | 33 +++-------
 .../fs/s3a/TestS3AAWSCredentialsProvider.java   |  4 +-
 5 files changed, 13 insertions(+), 108 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 03e4996..4f37c65 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -955,13 +955,8 @@
         configuration of AWS access key ID and secret access key in
         environment variables named AWS_ACCESS_KEY_ID and
         AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-    3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
-        a shared instance of
-        com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
-        SDK, which supports use of instance profile credentials if running
-        in an EC2 VM.  Using this shared instance potentially reduces load
-        on the EC2 instance metadata service for multi-threaded
-        applications.
+    3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+        of instance profile credentials if running in an EC2 VM.
   </description>
 </property>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6a11699..5ff9321 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -339,15 +339,9 @@ public final class S3AUtils {
       credentials.add(new BasicAWSCredentialsProvider(
               creds.getUser(), creds.getPassword()));
       credentials.add(new EnvironmentVariableCredentialsProvider());
-      credentials.add(
-          SharedInstanceProfileCredentialsProvider.getInstance());
+      credentials.add(InstanceProfileCredentialsProvider.getInstance());
     } else {
       for (Class<?> aClass : awsClasses) {
-        if (aClass == InstanceProfileCredentialsProvider.class) {
-          LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
-              SharedInstanceProfileCredentialsProvider.class.getName());
-          aClass = SharedInstanceProfileCredentialsProvider.class;
-        }
         credentials.add(createAWSCredentialProvider(conf, aClass));
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
deleted file mode 100644
index cbc0787..0000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A subclass of {@link InstanceProfileCredentialsProvider} that enforces
- * instantiation of only a single instance.
- * This credential provider calls the EC2 instance metadata service to obtain
- * credentials.  For highly multi-threaded applications, it's possible that
- * multiple instances call the service simultaneously and overwhelm it with
- * load.  The service handles this by throttling the client with an HTTP 429
- * response or forcibly terminating the connection.  Forcing use of a single
- * instance reduces load on the metadata service by allowing all threads to
- * share the credentials.  The base class is thread-safe, and there is nothing
- * that varies in the credentials across different instances of
- * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
- * instance is safe.
- *
- * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton.  After
- * Hadoop upgrades to that version or higher, it's likely that we can remove
- * this class.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public final class SharedInstanceProfileCredentialsProvider
-    extends InstanceProfileCredentialsProvider {
-
-  private static final SharedInstanceProfileCredentialsProvider INSTANCE =
-      new SharedInstanceProfileCredentialsProvider();
-
-  /**
-   * Returns the singleton instance.
-   *
-   * @return singleton instance
-   */
-  public static SharedInstanceProfileCredentialsProvider getInstance() {
-    return INSTANCE;
-  }
-
-  /**
-   * Default constructor, defined explicitly as private to enforce singleton.
-   */
-  private SharedInstanceProfileCredentialsProvider() {
-    super();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 18c0ceb..a3a0bb1 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
             configuration of AWS access key ID and secret access key in
             environment variables named AWS_ACCESS_KEY_ID and
             AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-        3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
-            a shared instance of
-            com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
-            SDK, which supports use of instance profile credentials if running
-            in an EC2 VM.  Using this shared instance potentially reduces load
-            on the EC2 instance metadata service for multi-threaded
-            applications.
+        3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+            of instance profile credentials if running in an EC2 VM.
       </description>
     </property>
 
@@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to
 obtain an AWS login from a different source in the system, including environment
 variables, JVM properties and configuration files.
 
-There are four AWS Credential Providers inside the `hadoop-aws` JAR:
+There are three AWS Credential Providers inside the `hadoop-aws` JAR:
 
 | classname | description |
 |-----------|-------------|
 | `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
 | `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
-| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service.  (See below.) |
 | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
 
 There are also many in the Amazon SDKs, in particular two which are automatically
@@ -425,24 +419,13 @@ set up in the authentication chain:
 | `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables |
 
 
-*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`*
+*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`*
 
 Applications running in EC2 may associate an IAM role with the VM and query the
 [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 for credentials to access S3.  Within the AWS SDK, this functionality is
-provided by `InstanceProfileCredentialsProvider`.  Heavily multi-threaded
-applications may trigger a high volume of calls to the instance metadata service
-and trigger throttling: either an HTTP 429 response or a forcible close of the
-connection.
-
-To mitigate against this problem, `hadoop-aws` ships with a variant of
-`InstanceProfileCredentialsProvider` called
-`SharedInstanceProfileCredentialsProvider`.  Using this ensures that all
-instances of S3A reuse the same instance profile credentials instead of issuing
-a large volume of redundant metadata service calls.  If
-`fs.s3a.aws.credentials.provider` refers to
-`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
-`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
+provided by `InstanceProfileCredentialsProvider`, which internally enforces a
+singleton instance in order to prevent throttling problem.
 
 *Session Credentials with `TemporaryAWSCredentialsProvider`*
 
@@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as
       <value>
       org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
       com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
-      org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+      com.amazonaws.auth.InstanceProfileCredentialsProvider
       </value>
     </property>
 
@@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2.
 ```xml
 <property>
   <name>fs.s3a.aws.credentials.provider</name>
-  <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
+  <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value>
 </property>
 ```
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 33740c8..82a8b84 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider {
         Arrays.asList(
             BasicAWSCredentialsProvider.class,
             EnvironmentVariableCredentialsProvider.class,
-            SharedInstanceProfileCredentialsProvider.class);
+            InstanceProfileCredentialsProvider.class);
     assertCredentialProviders(expectedClasses, list1);
     assertCredentialProviders(expectedClasses, list2);
     assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
@@ -128,7 +128,7 @@ public class TestS3AAWSCredentialsProvider {
     List<Class<? extends AWSCredentialsProvider>> expectedClasses =
         Arrays.asList(
             EnvironmentVariableCredentialsProvider.class,
-            SharedInstanceProfileCredentialsProvider.class,
+            InstanceProfileCredentialsProvider.class,
             AnonymousAWSCredentialsProvider.class);
     conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
     AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org