This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 70b6c155bcb HADOOP-18328. S3A to support S3 on Outposts (#4533)
70b6c155bcb is described below
commit 70b6c155bcb9d802a0f4c1132c75bfef202d1829
Author: suzu <s.suzugam...@gmail.com>
AuthorDate: Wed Aug 23 19:38:07 2023 +0900
HADOOP-18328. S3A to support S3 on Outposts (#4533)
Contributed by Sotetsu Suzugamine
---
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 10 +++++++--
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 24 +++++++++++++++++-----
2 files changed, 27 insertions(+), 7 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
index 0294f772290..a85f26223ff 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
@@ -26,7 +26,8 @@ import com.amazonaws.arn.Arn;
* Represents an Arn Resource, this can be an accesspoint or bucket.
*/
public final class ArnResource {
- private final static String ACCESSPOINT_ENDPOINT_FORMAT =
"s3-accesspoint.%s.amazonaws.com";
+ private final static String S3_ACCESSPOINT_ENDPOINT_FORMAT =
"s3-accesspoint.%s.amazonaws.com";
+ private final static String S3_OUTPOSTS_ACCESSPOINT_ENDPOINT_FORMAT =
"s3-outposts.%s.amazonaws.com";
/**
* Resource name.
@@ -69,6 +70,10 @@ public final class ArnResource {
this.accessPointRegionKey = String.format("accesspoint-%s", region);
}
+ private boolean isOutposts(){
+ return fullArn.contains("s3-outposts");
+ }
+
/**
* Resource name.
* @return resource name.
@@ -106,7 +111,8 @@ public final class ArnResource {
* @return resource endpoint.
*/
public String getEndpoint() {
- return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
+ String format = isOutposts() ? S3_OUTPOSTS_ACCESSPOINT_ENDPOINT_FORMAT :
S3_ACCESSPOINT_ENDPOINT_FORMAT;
+ return String.format(format, region);
}
/**
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
index 36381bf14b1..c881aac35d9 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
@@ -56,7 +56,7 @@ public class TestArnResource extends HadoopTestBase {
String region = testPair[0];
String partition = testPair[1];
- ArnResource resource = getArnResourceFrom(partition, region,
MOCK_ACCOUNT, accessPoint);
+ ArnResource resource = getArnResourceFrom(partition, "s3", region,
MOCK_ACCOUNT, accessPoint);
assertEquals("Access Point name does not match", accessPoint,
resource.getName());
assertEquals("Account Id does not match", MOCK_ACCOUNT,
resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion());
@@ -64,10 +64,10 @@ public class TestArnResource extends HadoopTestBase {
}
@Test
- public void makeSureEndpointHasTheCorrectFormat() {
+ public void makeSureS3EndpointHasTheCorrectFormat() {
// Access point (AP) endpoints are different from S3 bucket endpoints,
thus when using APs the
// endpoints for the client are modified. This test makes sure endpoint is
set up correctly.
- ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1",
MOCK_ACCOUNT,
+ ArnResource accessPoint = getArnResourceFrom("aws", "s3", "eu-west-1",
MOCK_ACCOUNT,
"test");
String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
@@ -76,6 +76,19 @@ public class TestArnResource extends HadoopTestBase {
.isEqualTo(expected);
}
+ @Test
+ public void makeSureS3OutpostsEndpointHasTheCorrectFormat() {
+ // Access point (AP) endpoints are different from S3 bucket endpoints,
thus when using APs the
+ // endpoints for the client are modified. This test makes sure endpoint is
set up correctly.
+ ArnResource accessPoint = getArnResourceFrom("aws", "s3-outposts",
"eu-west-1", MOCK_ACCOUNT,
+ "test");
+ String expected = "s3-outposts.eu-west-1.amazonaws.com";
+
+ Assertions.assertThat(accessPoint.getEndpoint())
+ .describedAs("Endpoint has invalid format. Access Point requests will
not work")
+ .isEqualTo(expected);
+ }
+
@Test
public void invalidARNsMustThrow() throws Exception {
describe("Using an invalid ARN format must throw when initializing an
ArnResource.");
@@ -87,15 +100,16 @@ public class TestArnResource extends HadoopTestBase {
/**
* Create an {@link ArnResource} from string components
* @param partition - partition for ARN
+ * @param service - service for ARN
* @param region - region for ARN
* @param accountId - accountId for ARN
* @param resourceName - ARN resource name
* @return ArnResource described by its properties
*/
- private ArnResource getArnResourceFrom(String partition, String region,
String accountId,
+ private ArnResource getArnResourceFrom(String partition, String service,
String region, String accountId,
String resourceName) {
// arn:partition:service:region:account-id:resource-type/resource-id
- String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition,
region, accountId,
+ String arn = String.format("arn:%s:%s:%s:%s:accesspoint/%s", partition,
service, region, accountId,
resourceName);
return ArnResource.accessPointFromArn(arn);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
