This is an automated email from the ASF dual-hosted git repository. rouazana pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 712f005be906c4eefdf3158960407b4a58dc5357 Author: RĂ©mi Kowalski <rkowal...@linagora.com> AuthorDate: Thu May 9 18:02:48 2019 +0200 JAMES-2725 Use injection for AwsS3Object storage for managing its thread pool lifecycle. --- .../blob/objectstorage/aws/AwsS3ObjectStorage.java | 30 +++++++++++++++++----- .../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 10 +++++++- .../ObjectStorageDependenciesModule.java | 11 +++++--- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java index fbf48a4..5454af0 100644 --- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java +++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java @@ -28,6 +28,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Supplier; +import javax.annotation.PreDestroy; +import javax.inject.Inject; + import org.apache.commons.io.FileUtils; import org.apache.james.blob.objectstorage.ContainerName; import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder; @@ -51,14 +54,14 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.google.inject.Module; public class AwsS3ObjectStorage { private static final Iterable<Module> JCLOUDS_MODULES = ImmutableSet.of(new SLF4JLoggingModule()); - public static final int MAX_THREADS = 5; - private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class)); + public static final int MAX_THREADS = 5; private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false; private static final int MAX_ERROR_RETRY = 5; private static final int FIRST_TRY = 0; @@ -73,11 +76,24 @@ public class AwsS3ObjectStorage { } } + private final ExecutorService executorService; + + @Inject + @VisibleForTesting + AwsS3ObjectStorage() { + executorService = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class)); + } + + @PreDestroy + public void tearDown() { + executorService.shutdownNow(); + } + public static ObjectStorageBlobsDAOBuilder.RequireContainerName daoBuilder(AwsS3AuthConfiguration configuration) { return ObjectStorageBlobsDAOBuilder.forBlobStore(new BlobStoreBuilder(configuration)); } - public static Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) { + public Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) { return Optional.of((blob) -> { File file = null; try { @@ -94,7 +110,7 @@ public class AwsS3ObjectStorage { }); } - private static void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) { + private void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) { try { put(containerName, configuration, blob, file); } catch (RuntimeException e) { @@ -106,7 +122,7 @@ public class AwsS3ObjectStorage { } } - private static void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) { + private void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) { try { PutObjectRequest request = new PutObjectRequest(containerName.value(), blob.getMetadata().getName(), @@ -120,7 +136,7 @@ public class AwsS3ObjectStorage { } } - private static TransferManager getTransferManager(AwsS3AuthConfiguration configuration) { + private TransferManager getTransferManager(AwsS3AuthConfiguration configuration) { ClientConfiguration clientConfiguration = getClientConfiguration(); AmazonS3 amazonS3 = getS3Client(configuration, clientConfiguration); @@ -128,7 +144,7 @@ public class AwsS3ObjectStorage { .standard() .withS3Client(amazonS3) .withMultipartUploadThreshold(MULTIPART_UPLOAD_THRESHOLD.getValue()) - .withExecutorFactory(() -> EXECUTOR_SERVICE) + .withExecutorFactory(() -> executorService) .withShutDownThreadPools(DO_NOT_SHUTDOWN_THREAD_POOL) .build(); } diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java index 6a0367e..3e71477 100644 --- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java +++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java @@ -28,6 +28,7 @@ import org.apache.james.blob.objectstorage.ContainerName; import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAO; import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder; import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOContract; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -37,9 +38,11 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont private ContainerName containerName; private AwsS3AuthConfiguration configuration; + private AwsS3ObjectStorage awsS3ObjectStorage; @BeforeEach void setUp(DockerAwsS3Container dockerAwsS3Container) { + awsS3ObjectStorage = new AwsS3ObjectStorage(); containerName = ContainerName.of(UUID.randomUUID().toString()); configuration = AwsS3AuthConfiguration.builder() .endpoint(dockerAwsS3Container.getEndpoint()) @@ -48,6 +51,11 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont .build(); } + @AfterEach + void tearDown() { + awsS3ObjectStorage.tearDown(); + } + @Override public ContainerName containerName() { return containerName; @@ -79,7 +87,7 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont .builder(configuration) .container(containerName) .blobIdFactory(new HashBlobId.Factory()) - .putBlob(AwsS3ObjectStorage.putBlob(containerName, configuration)); + .putBlob(awsS3ObjectStorage.putBlob(containerName, configuration)); assertBlobsDAOCanStoreAndRetrieve(builder); } diff --git a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java index 5e43388..a2b3953 100644 --- a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java +++ b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import javax.inject.Provider; import javax.inject.Singleton; import org.apache.commons.configuration.Configuration; @@ -64,12 +65,12 @@ public class ObjectStorageDependenciesModule extends AbstractModule { @Provides @Singleton - private ObjectStorageBlobsDAO buildObjectStore(ObjectStorageBlobConfiguration configuration, BlobId.Factory blobIdFactory) throws InterruptedException, ExecutionException, TimeoutException { + private ObjectStorageBlobsDAO buildObjectStore(ObjectStorageBlobConfiguration configuration, BlobId.Factory blobIdFactory, Provider<AwsS3ObjectStorage> awsS3ObjectStorageProvider) throws InterruptedException, ExecutionException, TimeoutException { ObjectStorageBlobsDAO dao = selectDaoBuilder(configuration) .container(configuration.getNamespace()) .blobIdFactory(blobIdFactory) .payloadCodec(configuration.getPayloadCodec()) - .putBlob(putBlob(blobIdFactory, configuration)) + .putBlob(putBlob(blobIdFactory, configuration, awsS3ObjectStorageProvider)) .build(); dao.createContainer(configuration.getNamespace()).block(Duration.ofMinutes(1)); return dao; @@ -85,12 +86,14 @@ public class ObjectStorageDependenciesModule extends AbstractModule { throw new IllegalArgumentException("unknown provider " + configuration.getProvider()); } - private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration) { + private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration, Provider<AwsS3ObjectStorage> awsS3ObjectStorageProvider) { switch (configuration.getProvider()) { case SWIFT: return Optional.empty(); case AWSS3: - return AwsS3ObjectStorage.putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration()); + return awsS3ObjectStorageProvider + .get() + .putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration()); } throw new IllegalArgumentException("unknown provider " + configuration.getProvider()); --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org