HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eed1645f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eed1645f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eed1645f Branch: refs/heads/HDFS-7285 Commit: eed1645fd3a513a2a82cf76b4063a4baf4e819f9 Parents: fd63337 Author: Steve Loughran <ste...@apache.org> Authored: Sun Mar 8 11:20:42 2015 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Mon Mar 9 13:11:27 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../org/apache/hadoop/fs/s3a/Constants.java | 6 +++++- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 20 ++++++++++++++------ .../src/site/markdown/tools/hadoop-aws/index.md | 10 +++++----- 4 files changed, 26 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed1645f/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 14cd75a..16002d5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1050,6 +1050,8 @@ Release 2.7.0 - UNRELEASED HADOOP-11674. oneByteBuf in CryptoInputStream and CryptoOutputStream should be non static. (Sean Busbey via yliu) + HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed1645f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index e7462dc..3486dfb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -18,8 +18,12 @@ package org.apache.hadoop.fs.s3a; - public class Constants { + // s3 access key + public static final String ACCESS_KEY = "fs.s3a.access.key"; + + // s3 secret key + public static final String SECRET_KEY = "fs.s3a.secret.key"; // number of simultaneous connections to s3 public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed1645f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 1a30d6f..91a606c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -32,8 +32,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.fs.s3.S3Credentials; - import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; @@ -159,12 +157,22 @@ public class S3AFileSystem extends FileSystem { this.getWorkingDirectory()); // Try to get our credentials or just connect anonymously - S3Credentials s3Credentials = new S3Credentials(); - s3Credentials.initialize(name, conf); + String accessKey = conf.get(ACCESS_KEY, null); + String secretKey = conf.get(SECRET_KEY, null); + + String userInfo = name.getUserInfo(); + if (userInfo != null) { + int index = userInfo.indexOf(':'); + if (index != -1) { + accessKey = userInfo.substring(0, index); + secretKey = userInfo.substring(index + 1); + } else { + accessKey = userInfo; + } + } AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( - new BasicAWSCredentialsProvider(s3Credentials.getAccessKey(), - s3Credentials.getSecretAccessKey()), + new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(), new AnonymousAWSCredentialsProvider() ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed1645f/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index bf62634..e0389c0 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -141,12 +141,12 @@ If you do any of these: change your credentials immediately! ### Authentication properties <property> - <name>fs.s3a.awsAccessKeyId</name> + <name>fs.s3a.access.key</name> <description>AWS access key ID. Omit for Role-based authentication.</description> </property> <property> - <name>fs.s3a.awsSecretAccessKey</name> + <name>fs.s3a.secret.key</name> <description>AWS secret key. Omit for Role-based authentication.</description> </property> @@ -411,13 +411,13 @@ Example: </property> <property> - <name>fs.s3a.awsAccessKeyId</name> + <name>fs.s3a.access.key</name> <description>AWS access key ID. Omit for Role-based authentication.</description> - <value>DONOTPCOMMITTHISKEYTOSCM</value> + <value>DONOTCOMMITTHISKEYTOSCM</value> </property> <property> - <name>fs.s3a.awsSecretAccessKey</name> + <name>fs.s3a.secret.key</name> <description>AWS secret key. Omit for Role-based authentication.</description> <value>DONOTEVERSHARETHISSECRETKEY!</value> </property>
