This is an automated email from the ASF dual-hosted git repository. rcordier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 5e52bc456ca12060d4342600466f1051605c3fd0 Author: Benoit Tellier <[email protected]> AuthorDate: Tue May 18 08:00:15 2021 +0700 JAMES-3028 Allow setting up S3 HTTP concurrency at the Netty level --- .../pages/distributed/configure/blobstore.adoc | 3 +++ .../aws/S3BlobStoreConfiguration.java | 24 +++++++++++++++++++--- .../blob/objectstorage/aws/S3BlobStoreDAO.java | 2 +- .../S3BlobStoreConfigurationReader.java | 3 +++ src/site/xdoc/server/config-blobstore.xml | 3 +++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/modules/servers/pages/distributed/configure/blobstore.adoc b/docs/modules/servers/pages/distributed/configure/blobstore.adoc index c256b13..5611040 100644 --- a/docs/modules/servers/pages/distributed/configure/blobstore.adoc +++ b/docs/modules/servers/pages/distributed/configure/blobstore.adoc @@ -109,6 +109,9 @@ Maximum size of stored objects expressed in bytes. | objectstorage.s3.secretKey | https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[S3 access key secret] + +| objectstorage.s3.http.concurrency +| Allow setting the number of concurrent HTTP requests allowed by the Netty driver. |=== ==== Buckets Configuration diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java index d0a1b7a..a456d4a 100644 --- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java +++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java @@ -51,6 +51,7 @@ public class S3BlobStoreConfiguration { private Optional<BucketName> defaultBucketName; private Optional<String> bucketPrefix; + private Optional<Integer> httpConcurrency; private Region region; public ReadyToBuild(AwsS3AuthConfiguration specificAuthConfiguration, Region region) { @@ -58,6 +59,7 @@ public class S3BlobStoreConfiguration { this.region = region; this.defaultBucketName = Optional.empty(); this.bucketPrefix = Optional.empty(); + this.httpConcurrency = Optional.empty(); } public ReadyToBuild defaultBucketName(Optional<BucketName> defaultBucketName) { @@ -80,27 +82,37 @@ public class S3BlobStoreConfiguration { return this; } + public ReadyToBuild httpConcurrency(Optional<Integer> httpConcurrency) { + this.httpConcurrency = httpConcurrency; + return this; + } + public S3BlobStoreConfiguration build() { - return new S3BlobStoreConfiguration(bucketPrefix, defaultBucketName, region, specificAuthConfiguration); + return new S3BlobStoreConfiguration(bucketPrefix, defaultBucketName, region, specificAuthConfiguration, httpConcurrency.orElse(DEFAULT_HTTP_CONCURRENCY)); } } } + public static int DEFAULT_HTTP_CONCURRENCY = 100; + private final Region region; private final AwsS3AuthConfiguration specificAuthConfiguration; private final Optional<BucketName> namespace; private final Optional<String> bucketPrefix; + private final int httpConcurrency; @VisibleForTesting S3BlobStoreConfiguration(Optional<String> bucketPrefix, Optional<BucketName> namespace, Region region, - AwsS3AuthConfiguration specificAuthConfiguration) { + AwsS3AuthConfiguration specificAuthConfiguration, + int httpConcurrency) { this.bucketPrefix = bucketPrefix; this.namespace = namespace; this.region = region; this.specificAuthConfiguration = specificAuthConfiguration; + this.httpConcurrency = httpConcurrency; } public Optional<BucketName> getNamespace() { @@ -115,6 +127,10 @@ public class S3BlobStoreConfiguration { return bucketPrefix; } + public int getHttpConcurrency() { + return httpConcurrency; + } + public Region getRegion() { return region; } @@ -127,6 +143,7 @@ public class S3BlobStoreConfiguration { return Objects.equals(this.namespace, that.namespace) && Objects.equals(this.bucketPrefix, that.bucketPrefix) && Objects.equals(this.region, that.region) + && Objects.equals(this.httpConcurrency, that.httpConcurrency) && Objects.equals(this.specificAuthConfiguration, that.specificAuthConfiguration); } return false; @@ -134,13 +151,14 @@ public class S3BlobStoreConfiguration { @Override public final int hashCode() { - return Objects.hash(namespace, bucketPrefix, specificAuthConfiguration); + return Objects.hash(namespace, bucketPrefix, httpConcurrency, specificAuthConfiguration); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("namespace", namespace) + .add("httpConcurrency", httpConcurrency) .add("bucketPrefix", bucketPrefix) .add("region", region) .add("specificAuthConfiguration", specificAuthConfiguration) diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java index d228aee..0a12fea 100644 --- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java +++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java @@ -97,7 +97,7 @@ public class S3BlobStoreDAO implements BlobStoreDAO, Startable, Closeable { .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(authConfiguration.getAccessKeyId(), authConfiguration.getSecretKey()))) .httpClientBuilder(NettyNioAsyncHttpClient.builder() - .maxConcurrency(100) + .maxConcurrency(configuration.getHttpConcurrency()) .maxPendingConnectionAcquires(10_000)) .endpointOverride(authConfiguration.getEndpoint()) .region(configuration.getRegion().asAws()) diff --git a/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java b/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java index 418c7a7..44b74f6 100644 --- a/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java +++ b/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java @@ -33,8 +33,10 @@ public class S3BlobStoreConfigurationReader { private static final String OBJECTSTORAGE_NAMESPACE = "objectstorage.namespace"; private static final String OBJECTSTORAGE_BUCKET_PREFIX = "objectstorage.bucketPrefix"; private static final String OBJECTSTORAGE_S3_REGION = "objectstorage.s3.region"; + private static final String OBJECTSTORAGE_S3_HTTP_CONCURRENCY = "objectstorage.s3.http.concurrency"; public static S3BlobStoreConfiguration from(Configuration configuration) throws ConfigurationException { + Optional<Integer> httpConcurrency = Optional.ofNullable(configuration.getInteger(OBJECTSTORAGE_S3_HTTP_CONCURRENCY, null)); Optional<String> namespace = Optional.ofNullable(configuration.getString(OBJECTSTORAGE_NAMESPACE, null)); Optional<String> bucketPrefix = Optional.ofNullable(configuration.getString(OBJECTSTORAGE_BUCKET_PREFIX, null)); Region region = Optional.ofNullable(configuration.getString(OBJECTSTORAGE_S3_REGION, null)) @@ -46,6 +48,7 @@ public class S3BlobStoreConfigurationReader { .region(region) .defaultBucketName(namespace.map(BucketName::of)) .bucketPrefix(bucketPrefix) + .httpConcurrency(httpConcurrency) .build(); } diff --git a/src/site/xdoc/server/config-blobstore.xml b/src/site/xdoc/server/config-blobstore.xml index 15ae7b2..513f8ca 100644 --- a/src/site/xdoc/server/config-blobstore.xml +++ b/src/site/xdoc/server/config-blobstore.xml @@ -149,6 +149,9 @@ generate salt with : openssl rand -hex 16 <dt><strong>objectstorage.s3.secretKey</strong></dt> <dd><a href="https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys";>S3 access key secret</a></dd> + + <dt><strong>objectstorage.s3.http.concurrency</strong></dt> + <dd>Allow setting the number of concurrent HTTP requests allowed by the Netty driver.</dd> </dl> </subsection> </subsection> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
