HADOOP-14248. Retire SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a631172e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a631172e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a631172e Branch: refs/heads/HDFS-10467 Commit: a631172ed9ae5ee1bda809f602eaed68e00614fa Parents: e2d6656 Author: Chris Nauroth <cnaur...@apache.org> Authored: Wed Apr 12 10:02:13 2017 -0700 Committer: Inigo <inigo...@apache.org> Committed: Mon Apr 17 11:17:02 2017 -0700 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 9 +-- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 8 +-- ...haredInstanceProfileCredentialsProvider.java | 67 -------------------- .../src/site/markdown/tools/hadoop-aws/index.md | 33 +++------- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 4 +- 5 files changed, 13 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a631172e/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 03e4996..4f37c65 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -955,13 +955,8 @@ configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/a631172e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 6a11699..5ff9321 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -339,15 +339,9 @@ public final class S3AUtils { credentials.add(new BasicAWSCredentialsProvider( creds.getUser(), creds.getPassword())); credentials.add(new EnvironmentVariableCredentialsProvider()); - credentials.add( - SharedInstanceProfileCredentialsProvider.getInstance()); + credentials.add(InstanceProfileCredentialsProvider.getInstance()); } else { for (Class<?> aClass : awsClasses) { - if (aClass == InstanceProfileCredentialsProvider.class) { - LOG.debug("Found {}, but will use {} instead.", aClass.getName(), - SharedInstanceProfileCredentialsProvider.class.getName()); - aClass = SharedInstanceProfileCredentialsProvider.class; - } credentials.add(createAWSCredentialProvider(conf, aClass)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a631172e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java deleted file mode 100644 index cbc0787..0000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a; - -import com.amazonaws.auth.InstanceProfileCredentialsProvider; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * A subclass of {@link InstanceProfileCredentialsProvider} that enforces - * instantiation of only a single instance. - * This credential provider calls the EC2 instance metadata service to obtain - * credentials. For highly multi-threaded applications, it's possible that - * multiple instances call the service simultaneously and overwhelm it with - * load. The service handles this by throttling the client with an HTTP 429 - * response or forcibly terminating the connection. Forcing use of a single - * instance reduces load on the metadata service by allowing all threads to - * share the credentials. The base class is thread-safe, and there is nothing - * that varies in the credentials across different instances of - * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton - * instance is safe. - * - * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After - * Hadoop upgrades to that version or higher, it's likely that we can remove - * this class. - */ -@InterfaceAudience.Private -@InterfaceStability.Stable -public final class SharedInstanceProfileCredentialsProvider - extends InstanceProfileCredentialsProvider { - - private static final SharedInstanceProfileCredentialsProvider INSTANCE = - new SharedInstanceProfileCredentialsProvider(); - - /** - * Returns the singleton instance. - * - * @return singleton instance - */ - public static SharedInstanceProfileCredentialsProvider getInstance() { - return INSTANCE; - } - - /** - * Default constructor, defined explicitly as private to enforce singleton. - */ - private SharedInstanceProfileCredentialsProvider() { - super(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a631172e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 18c0ceb..a3a0bb1 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. </description> </property> @@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to obtain an AWS login from a different source in the system, including environment variables, JVM properties and configuration files. -There are four AWS Credential Providers inside the `hadoop-aws` JAR: +There are three AWS Credential Providers inside the `hadoop-aws` JAR: | classname | description | |-----------|-------------| | `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials | | `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials | -| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service. (See below.) | | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login | There are also many in the Amazon SDKs, in particular two which are automatically @@ -425,24 +419,13 @@ set up in the authentication chain: | `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables | -*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`* +*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`* Applications running in EC2 may associate an IAM role with the VM and query the [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for credentials to access S3. Within the AWS SDK, this functionality is -provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded -applications may trigger a high volume of calls to the instance metadata service -and trigger throttling: either an HTTP 429 response or a forcible close of the -connection. - -To mitigate against this problem, `hadoop-aws` ships with a variant of -`InstanceProfileCredentialsProvider` called -`SharedInstanceProfileCredentialsProvider`. Using this ensures that all -instances of S3A reuse the same instance profile credentials instead of issuing -a large volume of redundant metadata service calls. If -`fs.s3a.aws.credentials.provider` refers to -`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses -`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead. +provided by `InstanceProfileCredentialsProvider`, which internally enforces a +singleton instance in order to prevent throttling problem. *Session Credentials with `TemporaryAWSCredentialsProvider`* @@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as <value> org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, com.amazonaws.auth.EnvironmentVariableCredentialsProvider, - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider </value> </property> @@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2. ```xml <property> <name>fs.s3a.aws.credentials.provider</name> - <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value> + <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value> </property> ``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/a631172e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 33740c8..82a8b84 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider { Arrays.asList( BasicAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class); + InstanceProfileCredentialsProvider.class); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2), @@ -128,7 +128,7 @@ public class TestS3AAWSCredentialsProvider { List<Class<? extends AWSCredentialsProvider>> expectedClasses = Arrays.asList( EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class, + InstanceProfileCredentialsProvider.class, AnonymousAWSCredentialsProvider.class); conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet( --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
