This is an automated email from the ASF dual-hosted git repository.
jtuglu1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 8308a662874 perf: Streamline s3 backend (#19394)
8308a662874 is described below
commit 8308a662874667b49e227a3422742922a08889e3
Author: jtuglu1 <[email protected]>
AuthorDate: Mon May 18 18:57:43 2026 -0700
perf: Streamline s3 backend (#19394)
S3 achieve strong read-after-write consistency in 2020. The current s3
backend architecture assumes a prior consistency model and therefore does some
redundant calls which are both slow and costly.
---
.../druid/storage/s3/S3DataSegmentMover.java | 125 +++++++++------------
.../druid/storage/s3/S3DataSegmentPuller.java | 23 +---
.../java/org/apache/druid/storage/s3/S3Utils.java | 18 +--
.../storage/s3/ServerSideEncryptingAmazonS3.java | 12 ++
.../druid/storage/s3/S3DataSegmentMoverTest.java | 52 +++++++++
.../druid/storage/s3/S3DataSegmentPullerTest.java | 9 --
.../druid/storage/s3/S3DataSegmentPusherTest.java | 50 +--------
.../apache/druid/storage/s3/S3TaskLogsTest.java | 19 ++--
8 files changed, 136 insertions(+), 172 deletions(-)
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentMover.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentMover.java
index cd6675c5b42..fe266cd72d1 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentMover.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentMover.java
@@ -24,8 +24,6 @@ import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
-import org.apache.druid.java.util.common.IOE;
-import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.MapUtils;
import org.apache.druid.java.util.common.RetryUtils;
import org.apache.druid.java.util.common.StringUtils;
@@ -35,13 +33,13 @@ import org.apache.druid.segment.loading.DataSegmentPusher;
import org.apache.druid.segment.loading.SegmentLoadingException;
import org.apache.druid.timeline.DataSegment;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
-import software.amazon.awssdk.services.s3.model.ObjectStorageClass;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.StorageClass;
-import java.io.IOException;
import java.nio.file.Paths;
import java.util.Map;
@@ -163,10 +161,10 @@ public class S3DataSegmentMover implements
DataSegmentMover
targetS3Path
);
try {
- selfCheckingMove(s3Bucket, targetS3Bucket, s3Path, targetS3Path,
copyMsg);
+ moveObject(s3Bucket, targetS3Bucket, s3Path, targetS3Path,
copyMsg);
return null;
}
- catch (S3Exception | IOException | SegmentLoadingException e) {
+ catch (S3Exception | SegmentLoadingException e) {
log.info(e, "Error while trying to move " + copyMsg);
throw e;
}
@@ -185,90 +183,75 @@ public class S3DataSegmentMover implements
DataSegmentMover
}
/**
- * Copies an object and after that checks that the object is present at the
target location, via a separate API call.
- * If it is not, an exception is thrown, and the object is not deleted at
the old location. This "paranoic" check
- * is added after it was observed that S3 may report a successful move, and
the object is not found at the target
- * location.
+ * Copies an S3 object to a target location and deletes the source.
+ * S3 has been strongly consistent since December 2020, so no post-copy
existence check is needed.
*/
- private void selfCheckingMove(
+ private void moveObject(
String s3Bucket,
String targetS3Bucket,
String s3Path,
String targetS3Path,
String copyMsg
- ) throws IOException, SegmentLoadingException
+ ) throws SegmentLoadingException
{
if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) {
log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket,
s3Path);
return;
}
final ServerSideEncryptingAmazonS3 s3Client = this.s3ClientSupplier.get();
- if (s3Client.doesObjectExist(s3Bucket, s3Path)) {
- ListObjectsV2Request request = ListObjectsV2Request.builder()
- .bucket(s3Bucket)
- .prefix(s3Path)
- .maxKeys(1)
- .build();
- final ListObjectsV2Response listResult = s3Client.listObjectsV2(request);
- // Using contents().size() instead of keyCount as, in some cases
- // it is observed that even though the contents returns some data
- // keyCount is still zero.
- if (listResult.contents().size() == 0) {
- // should never happen
- throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path);
- }
- final S3Object objectSummary = listResult.contents().get(0);
- if (objectSummary.storageClass() != null &&
- objectSummary.storageClass().equals(ObjectStorageClass.GLACIER)) {
- throw S3Exception.builder()
- .message(StringUtils.format(
- "Cannot move file[s3://%s/%s] of storage class glacier,
skipping.",
- s3Bucket,
- s3Path
- ))
- .build();
- } else {
- log.info("Moving file %s", copyMsg);
- CopyObjectRequest.Builder copyRequestBuilder =
CopyObjectRequest.builder()
- .sourceBucket(s3Bucket)
- .sourceKey(s3Path)
- .destinationBucket(targetS3Bucket)
- .destinationKey(targetS3Path);
- if (!config.getDisableAcl()) {
- final String headerValue = S3Utils.grantFullControlHeaderValue(
- S3Utils.grantFullControlToBucketOwner(s3Client, targetS3Bucket)
+ final HeadObjectResponse sourceMetadata;
+ try {
+ sourceMetadata = s3Client.getObjectMetadata(s3Bucket, s3Path);
+ }
+ catch (S3Exception e) {
+ if (e.statusCode() == 404) {
+ // Source is gone; succeed silently if it already landed at the target.
+ if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
+ log.info(
+ "Not moving file [s3://%s/%s], already present in target
location [s3://%s/%s]",
+ s3Bucket,
+ s3Path,
+ targetS3Bucket,
+ targetS3Path
);
- if (headerValue != null) {
- copyRequestBuilder.grantFullControl(headerValue);
- }
- }
- s3Client.copyObject(copyRequestBuilder);
- if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
- throw new IOE(
- "After copy was reported as successful the file doesn't exist in
the target location [%s]",
+ } else {
+ throw new SegmentLoadingException(
+ "Unable to move file %s, not present in either source or target
location",
copyMsg
);
}
- deleteWithRetriesSilent(s3Bucket, s3Path);
- log.debug("Finished moving file %s", copyMsg);
+ return;
}
- } else {
- // ensure object exists in target location
- if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
- log.info(
- "Not moving file [s3://%s/%s], already present in target location
[s3://%s/%s]",
- s3Bucket,
- s3Path,
- targetS3Bucket,
- targetS3Path
- );
- } else {
- throw new SegmentLoadingException(
- "Unable to move file %s, not present in either source or target
location",
- copyMsg
- );
+ throw e;
+ }
+ final StorageClass sc = sourceMetadata.storageClass();
+ if (StorageClass.GLACIER.equals(sc) ||
StorageClass.DEEP_ARCHIVE.equals(sc)) {
+ throw S3Exception.builder()
+ .message(StringUtils.format(
+ "Cannot move file[s3://%s/%s] of storage class [%s], skipping.",
+ s3Bucket,
+ s3Path,
+ sc
+ ))
+ .build();
+ }
+ log.info("Moving file %s", copyMsg);
+ CopyObjectRequest.Builder copyRequestBuilder = CopyObjectRequest.builder()
+ .sourceBucket(s3Bucket)
+ .sourceKey(s3Path)
+ .destinationBucket(targetS3Bucket)
+ .destinationKey(targetS3Path);
+ if (!config.getDisableAcl()) {
+ final String headerValue = S3Utils.grantFullControlHeaderValue(
+ s3Client.getBucketOwnerGrant(targetS3Bucket)
+ );
+ if (headerValue != null) {
+ copyRequestBuilder.grantFullControl(headerValue);
}
}
+ s3Client.copyObject(copyRequestBuilder);
+ deleteWithRetriesSilent(s3Bucket, s3Path);
+ log.debug("Finished moving file %s", copyMsg);
}
private void deleteWithRetriesSilent(final String s3Bucket, final String
s3Path)
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentPuller.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentPuller.java
index 797bb7fcb1c..d133256c49b 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentPuller.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentPuller.java
@@ -87,9 +87,6 @@ public class S3DataSegmentPuller implements URIDataPuller
FileUtils.mkdirp(outDir);
if (CompressionUtils.isZip(s3Coords.getPath())) {
- if (!isObjectInBucket(s3Coords)) {
- throw new SegmentLoadingException("IndexFile[%s] does not exist.",
s3Coords);
- }
final URI uri = s3Coords.toUri(S3StorageDruidModule.SCHEME);
final ByteSource byteSource = getByteSource(uri);
final FileUtils.FileCopyResult result = CompressionUtils.unzip(
@@ -101,9 +98,6 @@ public class S3DataSegmentPuller implements URIDataPuller
log.info("Loaded %d bytes from [%s] to [%s]", result.size(),
s3Coords.toString(), outDir.getAbsolutePath());
return result;
} else if (CompressionUtils.isGz(s3Coords.getPath())) {
- if (!isObjectInBucket(s3Coords)) {
- throw new SegmentLoadingException("IndexFile[%s] does not exist.",
s3Coords);
- }
final URI uri = s3Coords.toUri(S3StorageDruidModule.SCHEME);
final ByteSource byteSource = getByteSource(uri);
final String fname = Files.getNameWithoutExtension(uri.getPath());
@@ -235,6 +229,9 @@ public class S3DataSegmentPuller implements URIDataPuller
};
}
catch (SdkException e) {
+ if (e instanceof S3Exception && ((S3Exception) e).statusCode() ==
404) {
+ throw new IOE(e, "IndexFile[s3://%s/%s] does not exist",
coords.getBucket(), coords.getPath());
+ }
throw new IOE(e, "Could not load S3 URI [%s]", uri);
}
}
@@ -318,18 +315,4 @@ public class S3DataSegmentPuller implements URIDataPuller
}
}
- private boolean isObjectInBucket(final CloudObjectLocation coords) throws
SegmentLoadingException
- {
- try {
- return S3Utils.retryS3Operation(
- () -> S3Utils.isObjectInBucketIgnoringPermission(s3Client,
coords.getBucket(), coords.getPath())
- );
- }
- catch (S3Exception | IOException e) {
- throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords);
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
}
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
index 0feeb655fdf..2cac95a200b 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
@@ -43,11 +43,9 @@ import software.amazon.awssdk.services.s3.model.Grant;
import software.amazon.awssdk.services.s3.model.Grantee;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
-import software.amazon.awssdk.services.s3.model.Permission;
import software.amazon.awssdk.services.s3.model.S3Error;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
-import software.amazon.awssdk.services.s3.model.Type;
import javax.annotation.Nullable;
import java.io.File;
@@ -240,20 +238,6 @@ public class S3Utils
) + "/";
}
- static Grant grantFullControlToBucketOwner(ServerSideEncryptingAmazonS3
s3Client, String bucket)
- {
- final String ownerId = s3Client.getBucketAcl(bucket).owner().id();
- return Grant
- .builder()
- .grantee(Grantee
- .builder()
- .type(Type.CANONICAL_USER)
- .id(ownerId)
- .build())
- .permission(Permission.FULL_CONTROL)
- .build();
- }
-
/**
* Builds the header value for {@code x-amz-grant-full-control}.
*
@@ -427,7 +411,7 @@ public class S3Utils
)
{
log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key);
- service.upload(bucket, key, file, disableAcl ? null :
S3Utils.grantFullControlToBucketOwner(service, bucket));
+ service.upload(bucket, key, file, disableAcl ? null :
service.getBucketOwnerGrant(bucket));
}
/**
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
index d59124ecba2..b8b76d0cf5a 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
@@ -40,13 +40,16 @@ import
software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.Grant;
+import software.amazon.awssdk.services.s3.model.Grantee;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.Permission;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.Type;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
@@ -139,6 +142,15 @@ public class ServerSideEncryptingAmazonS3
return s3Client.getBucketAcl(builder -> builder.bucket(bucket));
}
+ public Grant getBucketOwnerGrant(String bucket)
+ {
+ final String ownerId = getBucketAcl(bucket).owner().id();
+ return Grant.builder()
+
.grantee(Grantee.builder().type(Type.CANONICAL_USER).id(ownerId).build())
+ .permission(Permission.FULL_CONTROL)
+ .build();
+ }
+
public HeadObjectResponse getObjectMetadata(String bucket, String key)
{
HeadObjectRequest.Builder requestBuilder = HeadObjectRequest.builder()
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
index c504fbe7e04..4a9e8b4b9b1 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
@@ -38,6 +38,7 @@ import
software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import software.amazon.awssdk.services.s3.model.Grant;
import software.amazon.awssdk.services.s3.model.Grantee;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.Owner;
@@ -130,6 +131,45 @@ public class S3DataSegmentMoverTest
Assert.assertFalse(mockS3Client.didMove());
}
+ @Test
+ public void testMoveNoopWhenHeadReturnsGeneric404() throws Exception
+ {
+ final MockAmazonS3Client mockS3Client = new MockAmazonS3Client()
+ {
+ @Override
+ public HeadObjectResponse getObjectMetadata(String bucketName, String
key)
+ {
+ if (doesObjectExist(bucketName, key)) {
+ return
HeadObjectResponse.builder().storageClass(StorageClass.STANDARD).build();
+ }
+ throw (S3Exception) S3Exception.builder().statusCode(404).build();
+ }
+ };
+ final S3DataSegmentMover mover = new S3DataSegmentMover(
+ Suppliers.ofInstance(mockS3Client),
+ new S3DataSegmentPusherConfig()
+ );
+
+ mockS3Client.putObject(
+ "archive",
+
"targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip"
+ );
+
+ final DataSegment movedSegment = mover.move(
+ SOURCE_SEGMENT,
+ ImmutableMap.of("baseKey", "targetBaseKey", "bucket", "archive")
+ );
+
+ final Map<String, Object> targetLoadSpec = movedSegment.getLoadSpec();
+
+ Assert.assertEquals(
+
"targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip",
+ MapUtils.getString(targetLoadSpec, "key")
+ );
+ Assert.assertEquals("archive", MapUtils.getString(targetLoadSpec,
"bucket"));
+ Assert.assertFalse(mockS3Client.didMove());
+ }
+
@Test(expected = SegmentLoadingException.class)
public void testMoveException() throws Exception
{
@@ -250,6 +290,18 @@ public class S3DataSegmentMoverTest
return (objects != null && objects.contains(objectKey));
}
+ @Override
+ public HeadObjectResponse getObjectMetadata(String bucketName, String key)
+ {
+ if (doesObjectExist(bucketName, key)) {
+ return
HeadObjectResponse.builder().storageClass(StorageClass.STANDARD).build();
+ }
+ throw (S3Exception) S3Exception.builder()
+
.awsErrorDetails(AwsErrorDetails.builder().errorCode("NoSuchKey").build())
+ .statusCode(404)
+ .build();
+ }
+
@Override
public ListObjectsV2Response listObjectsV2(ListObjectsV2Request
listObjectsV2Request)
{
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPullerTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPullerTest.java
index bc1b9f6ad76..289c60219af 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPullerTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPullerTest.java
@@ -101,9 +101,6 @@ public class S3DataSegmentPullerTest
final File tmpDir = temporaryFolder.newFolder("gzTestDir");
- EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(bucket),
EasyMock.eq(key)))
- .andReturn(true)
- .once();
EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(key)))
.andAnswer(() -> new ResponseInputStream<>(
GetObjectResponse.builder().lastModified(Instant.ofEpochMilli(0)).build(),
@@ -152,9 +149,6 @@ public class S3DataSegmentPullerTest
.build())
.statusCode(404)
.build();
- EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(bucket),
EasyMock.eq(key)))
- .andReturn(true)
- .once();
EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(key)))
.andThrow(exception)
.once();
@@ -201,9 +195,6 @@ public class S3DataSegmentPullerTest
.build())
.statusCode(503)
.build();
- EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(bucket),
EasyMock.eq(key)))
- .andReturn(true)
- .once();
EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(key)))
.andThrow(exception)
.once();
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java
index 036df9446fa..6bde494d900 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java
@@ -33,10 +33,8 @@ import org.junit.Test;
import org.junit.internal.matchers.ThrowableMessageMatcher;
import org.junit.rules.TemporaryFolder;
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
-import software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import software.amazon.awssdk.services.s3.model.Grant;
import software.amazon.awssdk.services.s3.model.Grantee;
-import software.amazon.awssdk.services.s3.model.Owner;
import software.amazon.awssdk.services.s3.model.Permission;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.Type;
@@ -96,23 +94,11 @@ public class S3DataSegmentPusherTest
{
ServerSideEncryptingAmazonS3 s3Client =
EasyMock.createStrictMock(ServerSideEncryptingAmazonS3.class);
- Owner owner = Owner.builder()
- .id("ownerId")
- .displayName("owner")
- .build();
- Grantee grantee = Grantee.builder()
- .id(owner.id())
- .type(Type.CANONICAL_USER)
- .build();
Grant grant = Grant.builder()
- .grantee(grantee)
+
.grantee(Grantee.builder().id("ownerId").type(Type.CANONICAL_USER).build())
.permission(Permission.FULL_CONTROL)
.build();
- final GetBucketAclResponse aclResponse = GetBucketAclResponse.builder()
- .owner(owner)
- .grants(grant)
- .build();
-
EasyMock.expect(s3Client.getBucketAcl(EasyMock.eq("bucket"))).andReturn(aclResponse).once();
+
EasyMock.expect(s3Client.getBucketOwnerGrant(EasyMock.eq("bucket"))).andReturn(grant).once();
s3Client.upload(EasyMock.anyString(), EasyMock.anyString(),
EasyMock.anyObject(File.class), EasyMock.anyObject(Grant.class));
EasyMock.expectLastCall().once();
@@ -136,23 +122,11 @@ public class S3DataSegmentPusherTest
{
ServerSideEncryptingAmazonS3 s3Client =
EasyMock.createStrictMock(ServerSideEncryptingAmazonS3.class);
- Owner owner = Owner.builder()
- .id("ownerId")
- .displayName("owner")
- .build();
- Grantee grantee = Grantee.builder()
- .id(owner.id())
- .type(Type.CANONICAL_USER)
- .build();
Grant grant = Grant.builder()
- .grantee(grantee)
+
.grantee(Grantee.builder().id("ownerId").type(Type.CANONICAL_USER).build())
.permission(Permission.FULL_CONTROL)
.build();
- final GetBucketAclResponse aclResponse = GetBucketAclResponse.builder()
- .owner(owner)
- .grants(grant)
- .build();
-
EasyMock.expect(s3Client.getBucketAcl(EasyMock.eq("bucket"))).andReturn(aclResponse).once();
+
EasyMock.expect(s3Client.getBucketOwnerGrant(EasyMock.eq("bucket"))).andReturn(grant).once();
s3Client.upload(EasyMock.anyString(), EasyMock.anyString(),
EasyMock.anyObject(File.class), EasyMock.anyObject(Grant.class));
EasyMock.expectLastCall().once();
@@ -174,23 +148,11 @@ public class S3DataSegmentPusherTest
.statusCode(400)
.build();
- Owner owner = Owner.builder()
- .id("ownerId")
- .displayName("owner")
- .build();
- Grantee grantee = Grantee.builder()
- .id(owner.id())
- .type(Type.CANONICAL_USER)
- .build();
Grant grant = Grant.builder()
- .grantee(grantee)
+
.grantee(Grantee.builder().id("ownerId").type(Type.CANONICAL_USER).build())
.permission(Permission.FULL_CONTROL)
.build();
- final GetBucketAclResponse aclResponse = GetBucketAclResponse.builder()
- .owner(owner)
- .grants(grant)
- .build();
-
EasyMock.expect(s3Client.getBucketAcl(EasyMock.eq("bucket"))).andReturn(aclResponse).once();
+
EasyMock.expect(s3Client.getBucketOwnerGrant(EasyMock.eq("bucket"))).andReturn(grant).once();
s3Client.upload(EasyMock.anyString(), EasyMock.anyString(),
EasyMock.anyObject(File.class), EasyMock.anyObject(Grant.class));
EasyMock.expectLastCall().andThrow(e).once();
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java
index f82f587f1ae..2516b62ba4c 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java
@@ -39,16 +39,16 @@ import org.junit.runner.RunWith;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
-import software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.Grant;
+import software.amazon.awssdk.services.s3.model.Grantee;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
-import software.amazon.awssdk.services.s3.model.Owner;
import software.amazon.awssdk.services.s3.model.Permission;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Type;
import javax.annotation.Nonnull;
import java.io.BufferedReader;
@@ -633,22 +633,19 @@ public class S3TaskLogsTest extends EasyMockSupport
private List<Grant> testPushInternal(boolean disableAcl, String ownerId,
String ownerDisplayName) throws Exception
{
List<Grant> capturedGrants = new ArrayList<>();
- Owner owner = Owner.builder()
- .id(ownerId)
- .displayName(ownerDisplayName)
- .build();
if (disableAcl) {
// When ACL is disabled, upload is called with null grant
s3Client.upload(EasyMock.anyString(), EasyMock.anyString(),
EasyMock.anyObject(File.class), EasyMock.isNull(Grant.class));
EasyMock.expectLastCall().once();
} else {
- // When ACL is enabled, getBucketAcl is called and a grant is created
- GetBucketAclResponse aclResponse = GetBucketAclResponse.builder()
- .owner(owner)
+ // When ACL is enabled, getBucketOwnerGrant is called and a grant is
created
+ Grant ownerGrant = Grant.builder()
+
.grantee(Grantee.builder().type(Type.CANONICAL_USER).id(ownerId).build())
+ .permission(Permission.FULL_CONTROL)
.build();
- EasyMock.expect(s3Client.getBucketAcl(TEST_BUCKET))
- .andReturn(aclResponse)
+ EasyMock.expect(s3Client.getBucketOwnerGrant(TEST_BUCKET))
+ .andReturn(ownerGrant)
.once();
Capture<Grant> grantCapture = Capture.newInstance(CaptureType.FIRST);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
