This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 542b4cbf4fdc7aac7c5d59e8879fea3063e2c963 Author: Tran Tien Duc <dt...@linagora.com> AuthorDate: Thu Jul 4 17:09:03 2019 +0700 JAMES-2806 ObjectStorage Swift save retry save() on non existing bucket --- server/blob/blob-objectstorage/pom.xml | 4 +++ .../objectstorage/StreamCompatibleBlobPutter.java | 34 ++++++++++++++++++++-- .../objectstorage/ObjectStorageBlobsDAOTest.java | 12 +++----- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/server/blob/blob-objectstorage/pom.xml b/server/blob/blob-objectstorage/pom.xml index c312e93..dabae94 100644 --- a/server/blob/blob-objectstorage/pom.xml +++ b/server/blob/blob-objectstorage/pom.xml @@ -87,6 +87,10 @@ <artifactId>guava</artifactId> </dependency> <dependency> + <groupId>io.projectreactor.addons</groupId> + <artifactId>reactor-extra</artifactId> + </dependency> + <dependency> <groupId>nl.jqno.equalsverifier</groupId> <artifactId>equalsverifier</artifactId> <scope>test</scope> diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java index f01bc34..1bac8e2 100644 --- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java +++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java @@ -19,6 +19,7 @@ package org.apache.james.blob.objectstorage; +import java.time.Duration; import java.util.function.Supplier; import org.apache.james.blob.api.BlobId; @@ -26,9 +27,21 @@ import org.apache.james.blob.api.BucketName; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.CopyOptions; +import org.jclouds.domain.Location; +import org.jclouds.http.HttpResponseException; + +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; +import reactor.retry.Retry; public class StreamCompatibleBlobPutter implements BlobPutter { - private final org.jclouds.blobstore.BlobStore blobStore; + + private static final int MAX_RETRIES = 3; + private static final Duration FIRST_BACK_OFF = Duration.ofMillis(100); + private static final Duration FOREVER = Duration.ofMillis(Long.MAX_VALUE); + private static final Location DEFAULT_LOCATION = null; + + private final BlobStore blobStore; public StreamCompatibleBlobPutter(BlobStore blobStore) { this.blobStore = blobStore; @@ -36,7 +49,14 @@ public class StreamCompatibleBlobPutter implements BlobPutter { @Override public void putDirectly(BucketName bucketName, Blob blob) { - blobStore.putBlob(bucketName.asString(), blob); + Mono.fromRunnable(() -> blobStore.putBlob(bucketName.asString(), blob)) + .publishOn(Schedulers.elastic()) + .retryWhen(Retry.onlyIf(retryContext -> needToCreateBucket(retryContext.exception(), bucketName)) + .exponentialBackoff(FIRST_BACK_OFF, FOREVER) + .withBackoffScheduler(Schedulers.elastic()) + .retryMax(MAX_RETRIES) + .doOnRetry(retryContext -> blobStore.createContainerInLocation(DEFAULT_LOCATION, bucketName.asString()))) + .block(); } @Override @@ -52,4 +72,14 @@ public class StreamCompatibleBlobPutter implements BlobPutter { blobStore.copyBlob(bucketNameAsString, from, bucketNameAsString, to, CopyOptions.NONE); blobStore.removeBlob(bucketNameAsString, from); } + + private boolean needToCreateBucket(Throwable throwable, BucketName bucketName) { + if (throwable instanceof HttpResponseException) { + HttpResponseException ex = (HttpResponseException) throwable; + return ex.getCommand().getCurrentRequest().getMethod().equals("PUT") + && !blobStore.containerExists(bucketName.asString()); + } + + return false; + } } \ No newline at end of file diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java index 12f11b8..a6321f7 100644 --- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java +++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java @@ -31,6 +31,7 @@ import java.util.UUID; import org.apache.commons.io.IOUtils; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BlobStore; +import org.apache.james.blob.api.BucketBlobStoreContract; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.HashBlobId; import org.apache.james.blob.api.MetricableBlobStore; @@ -45,7 +46,6 @@ import org.apache.james.blob.objectstorage.swift.UserHeaderName; import org.apache.james.blob.objectstorage.swift.UserName; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -56,7 +56,8 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @ExtendWith(DockerSwiftExtension.class) -public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract { +public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract, BucketBlobStoreContract { + private static final String BIG_STRING = Strings.repeat("big blob content", 10 * 1024); private static final TenantName TENANT_NAME = TenantName.of("test"); private static final UserName USER_NAME = UserName.of("tester"); @@ -98,6 +99,7 @@ public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract { @AfterEach void tearDown() { blobStore.deleteContainer(defaultBucketName.asString()); + blobStore.deleteContainer(CUSTOM.asString()); blobStore.getContext().close(); } @@ -111,12 +113,6 @@ public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract { return new HashBlobId.Factory(); } - @Override - @Disabled("JAMES-2806: delete bucket not implemented yet for ObjectStorage") - public void deleteBucketShouldPublishDeleteTimerMetrics() { - - } - @Test void createBucketShouldMakeTheContainerToExist() { BucketName bucketName = BucketName.of(UUID.randomUUID().toString()); --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org