[ https://issues.apache.org/jira/browse/HADOOP-14237?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15943066#comment-15943066 ]
ASF GitHub Bot commented on HADOOP-14237: ----------------------------------------- Github user steveloughran commented on a diff in the pull request: https://github.com/apache/hadoop/pull/207#discussion_r108144389 --- Diff: hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java --- @@ -58,6 +71,84 @@ public static SharedInstanceProfileCredentialsProvider getInstance() { return INSTANCE; } + private AWSCredentials readCredentialsFromHDFS() { + try { + FileSystem fs = FileSystem.get(new Configuration()); + BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(s3crednetialPath))); + String accessKey = br.readLine(); + String secretKey = br.readLine(); + String token = br.readLine(); + AWSCredentials credentials; + if (StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey)) { + // if there are no accessKey nor secretKey return null + return null; + } else if (StringUtils.isNotEmpty(token)) { + credentials = new BasicSessionCredentials(accessKey, secretKey, token); + } else { + credentials = new BasicAWSCredentials(accessKey, secretKey); + } + return credentials; + } catch (Exception e) { + return null; // ignore the read errors + // throw new AmazonServiceException("Failed reading S3 credentials from HDFS " + e.getStackTrace()); + } + } + + private void writeCredentialsToHDFS(AWSCredentials credentials) { + try { + // Simulate atomic write by creating a new s3credential file with random string suffix and rename to s3crednetialPath + Path newS3crednetialPath = new Path(s3crednetialPath.toUri() + RandomStringUtils.randomAlphanumeric(8)); + FileSystem fs = FileSystem.get(new Configuration()); + BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(newS3crednetialPath, true))); + String accessKey = credentials.getAWSAccessKeyId(); + String secretKey = credentials.getAWSSecretKey(); + String token = ""; + if (credentials instanceof BasicSessionCredentials) { + token = ((BasicSessionCredentials) credentials).getSessionToken(); + } + br.write(accessKey); + br.newLine(); + br.write(secretKey); + br.newLine(); + br.write(token); + br.newLine(); + br.close(); + fs.delete(s3crednetialPath, false); + fs.rename(newS3crednetialPath, s3crednetialPath); + } catch (Exception e) { + // ignore write errors + // throw new AmazonServiceException("Failed writing S3 credentials from HDFS " + e.getStackTrace()); + } + } + + @Override + public AWSCredentials getCredentials() { + for (int retry = 0; retry < maxRetries; retry++) { + try { + AWSCredentials newCredentials = super.getCredentials(); + // if this new credentials is different from HDFS write back + if (credentials == null || (!newCredentials.getAWSSecretKey().equals(credentials.getAWSSecretKey()))) { + credentials = newCredentials; + writeCredentialsToHDFS(credentials); + } + break; + } catch (Exception e) { --- End diff -- I't use our normal Retry logic here, consider some sleep + jitter if it really is caused by throttling > S3A Support Shared Instance Profile Credentials Across All Hadoop Nodes > ----------------------------------------------------------------------- > > Key: HADOOP-14237 > URL: https://issues.apache.org/jira/browse/HADOOP-14237 > Project: Hadoop Common > Issue Type: Bug > Components: fs/s3 > Affects Versions: 2.8.0, 3.0.0-alpha1, 3.0.0-alpha2, 2.8.1 > Environment: EC2, AWS > Reporter: Kazuyuki Tanimura > > When I run a large Hadoop cluster on EC2 instances with IAM Role, it fails > getting the instance profile credentials, eventually all jobs on the cluster > fail. Since a number of S3A clients (all mappers and reducers) try to get the > credentials, the AWS credential endpoint starts responding 5xx and 4xx error > codes. > SharedInstanceProfileCredentialsProvider.java is sort of trying to solve it, > but it still does not share the credentials with other EC2 nodes / JVM > processes. > This issue prevents users from creating Hadoop clusters on EC2 -- This message was sent by Atlassian JIRA (v6.3.15#6346) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org