[ https://issues.apache.org/jira/browse/HADOOP-14237?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15943068#comment-15943068 ]
ASF GitHub Bot commented on HADOOP-14237: ----------------------------------------- Github user steveloughran commented on a diff in the pull request: https://github.com/apache/hadoop/pull/207#discussion_r108144627 --- Diff: hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java --- @@ -58,6 +71,84 @@ public static SharedInstanceProfileCredentialsProvider getInstance() { return INSTANCE; } + private AWSCredentials readCredentialsFromHDFS() { + try { + FileSystem fs = FileSystem.get(new Configuration()); + BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(s3crednetialPath))); + String accessKey = br.readLine(); + String secretKey = br.readLine(); + String token = br.readLine(); + AWSCredentials credentials; + if (StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey)) { + // if there are no accessKey nor secretKey return null + return null; + } else if (StringUtils.isNotEmpty(token)) { + credentials = new BasicSessionCredentials(accessKey, secretKey, token); + } else { + credentials = new BasicAWSCredentials(accessKey, secretKey); + } + return credentials; + } catch (Exception e) { + return null; // ignore the read errors + // throw new AmazonServiceException("Failed reading S3 credentials from HDFS " + e.getStackTrace()); + } + } + + private void writeCredentialsToHDFS(AWSCredentials credentials) { + try { + // Simulate atomic write by creating a new s3credential file with random string suffix and rename to s3crednetialPath + Path newS3crednetialPath = new Path(s3crednetialPath.toUri() + RandomStringUtils.randomAlphanumeric(8)); + FileSystem fs = FileSystem.get(new Configuration()); + BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(newS3crednetialPath, true))); + String accessKey = credentials.getAWSAccessKeyId(); + String secretKey = credentials.getAWSSecretKey(); + String token = ""; + if (credentials instanceof BasicSessionCredentials) { --- End diff -- I would only allow session credentials to persist, so as to reduce risk of leakage of persistent secrets > S3A Support Shared Instance Profile Credentials Across All Hadoop Nodes > ----------------------------------------------------------------------- > > Key: HADOOP-14237 > URL: https://issues.apache.org/jira/browse/HADOOP-14237 > Project: Hadoop Common > Issue Type: Bug > Components: fs/s3 > Affects Versions: 2.8.0, 3.0.0-alpha1, 3.0.0-alpha2, 2.8.1 > Environment: EC2, AWS > Reporter: Kazuyuki Tanimura > > When I run a large Hadoop cluster on EC2 instances with IAM Role, it fails > getting the instance profile credentials, eventually all jobs on the cluster > fail. Since a number of S3A clients (all mappers and reducers) try to get the > credentials, the AWS credential endpoint starts responding 5xx and 4xx error > codes. > SharedInstanceProfileCredentialsProvider.java is sort of trying to solve it, > but it still does not share the credentials with other EC2 nodes / JVM > processes. > This issue prevents users from creating Hadoop clusters on EC2 -- This message was sent by Atlassian JIRA (v6.3.15#6346) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org