This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push: new c1c9c268f3 JAMES-2287 HashBlobId.Factory support encoding type (#1632) c1c9c268f3 is described below commit c1c9c268f3f42a18387b3a2bba2e08a3270eb956 Author: vttran <vtt...@linagora.com> AuthorDate: Mon Jul 10 09:01:01 2023 +0700 JAMES-2287 HashBlobId.Factory support encoding type (#1632) --- .../docs/modules/ROOT/pages/configure/jvm.adoc | 12 ++ .../java/org/apache/james/blob/api/HashBlobId.java | 33 +++++- .../org/apache/james/blob/api/HashBlobIdTest.java | 42 ++++++- server/blob/blob-s3/pom.xml | 5 + .../james/blob/objectstorage/aws/S3MinioTest.java | 123 +++++++++++++++++++++ src/site/xdoc/server/config-system.xml | 3 + 6 files changed, 214 insertions(+), 4 deletions(-) diff --git a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc index f0440ab4b2..d95acb2414 100644 --- a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc +++ b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/jvm.adoc @@ -53,3 +53,15 @@ james.protocols.mdc.hostname=false Optional. Boolean. Defaults to true. +== Change the encoding type used for the blobId + +By default, the blobId is encoded in base64 url. The property `james.blob.id.hash.encoding` allows to change the encoding type. +The support value are: base16, hex, base32, base32Hex, base64, base64Url. + +Ex in `jvm.properties` +---- +james.blob.id.hash.encoding=base16 +---- + +Optional. String. Defaults to base64Url. + diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java index e716df4040..55c1329e27 100644 --- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java +++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java @@ -20,7 +20,7 @@ package org.apache.james.blob.api; import java.io.IOException; -import java.util.Base64; +import java.util.Optional; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -29,11 +29,22 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; import com.google.common.io.ByteSource; public class HashBlobId implements BlobId { + private static final String HASH_BLOB_ID_ENCODING_TYPE_PROPERTY = "james.blob.id.hash.encoding"; + private static final BaseEncoding HASH_BLOB_ID_ENCODING_DEFAULT = BaseEncoding.base64Url(); public static class Factory implements BlobId.Factory { + private final BaseEncoding baseEncoding; + + public Factory() { + this.baseEncoding = Optional.ofNullable(System.getProperty(HASH_BLOB_ID_ENCODING_TYPE_PROPERTY)) + .map(Factory::baseEncodingFrom) + .orElse(HASH_BLOB_ID_ENCODING_DEFAULT); + } + @Override public HashBlobId forPayload(byte[] payload) { Preconditions.checkArgument(payload != null); @@ -51,7 +62,7 @@ public class HashBlobId implements BlobId { private HashBlobId base64(HashCode hashCode) { byte[] bytes = hashCode.asBytes(); - return new HashBlobId(Base64.getEncoder().encodeToString(bytes)); + return new HashBlobId(baseEncoding.encode(bytes)); } @Override @@ -59,6 +70,24 @@ public class HashBlobId implements BlobId { Preconditions.checkArgument(!Strings.isNullOrEmpty(id)); return new HashBlobId(id); } + + private static BaseEncoding baseEncodingFrom(String encodingType) { + switch (encodingType) { + case "base16": + case "hex": + return BaseEncoding.base16(); + case "base64": + return BaseEncoding.base64(); + case "base64Url": + return BaseEncoding.base64Url(); + case "base32": + return BaseEncoding.base32(); + case "base32Hex": + return BaseEncoding.base32Hex(); + default: + throw new IllegalArgumentException("Unknown encoding type: " + encodingType); + } + } } private final String id; diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java index 026a57d9f8..88fe109973 100644 --- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java +++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java @@ -23,16 +23,26 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; import org.apache.james.util.ClassLoaderUtils; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class HashBlobIdTest { private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory(); + @BeforeEach + void beforeEach(){ + System.clearProperty("james.blob.id.hash.encoding"); + } + @Test void shouldRespectBeanContract() { EqualsVerifier.forClass(HashBlobId.class).verify(); @@ -67,14 +77,42 @@ class HashBlobIdTest { void forPayloadShouldHashEmptyArray() { BlobId blobId = BLOB_ID_FACTORY.forPayload(new byte[0]); - assertThat(blobId.asString()).isEqualTo("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="); + assertThat(blobId.asString()).isEqualTo("47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU="); } @Test void forPayloadShouldHashArray() { BlobId blobId = BLOB_ID_FACTORY.forPayload("content".getBytes(StandardCharsets.UTF_8)); - assertThat(blobId.asString()).isEqualTo("7XACtDnprIRfIjV9giusFERzD722AW0+yUMil7nsn3M="); + assertThat(blobId.asString()).isEqualTo("7XACtDnprIRfIjV9giusFERzD722AW0-yUMil7nsn3M="); + } + + + @ParameterizedTest + @MethodSource("encodingTypeAndExpectedHash") + void forPayloadShouldSupportEncodingWhenConfigured(String encoding, String expectedHash) { + System.setProperty("james.blob.id.hash.encoding", encoding); + BlobId blobId = new HashBlobId.Factory().forPayload("content".getBytes(StandardCharsets.UTF_8)); + assertThat(blobId.asString()).isEqualTo(expectedHash); + } + + static Stream<Arguments> encodingTypeAndExpectedHash() { + return Stream.of( + Arguments.of("base16", "ED7002B439E9AC845F22357D822BAC1444730FBDB6016D3EC9432297B9EC9F73"), + Arguments.of("hex", "ED7002B439E9AC845F22357D822BAC1444730FBDB6016D3EC9432297B9EC9F73"), + Arguments.of("base32", "5VYAFNBZ5GWIIXZCGV6YEK5MCRCHGD55WYAW2PWJIMRJPOPMT5ZQ===="), + Arguments.of("base64", "7XACtDnprIRfIjV9giusFERzD722AW0+yUMil7nsn3M="), + Arguments.of("base64Url", "7XACtDnprIRfIjV9giusFERzD722AW0-yUMil7nsn3M="), + Arguments.of("base32", "5VYAFNBZ5GWIIXZCGV6YEK5MCRCHGD55WYAW2PWJIMRJPOPMT5ZQ===="), + Arguments.of("base32Hex", "TLO05D1PT6M88NP26LUO4ATC2H2763TTMO0MQFM98CH9FEFCJTPG====")); + } + + @Test + void newFactoryShouldFailWhenInvalidEncoding() { + System.setProperty("james.blob.id.hash.encoding", "invalid"); + assertThatThrownBy(HashBlobId.Factory::new) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unknown encoding type: invalid"); } @Test diff --git a/server/blob/blob-s3/pom.xml b/server/blob/blob-s3/pom.xml index e56fc0da9d..951301625f 100644 --- a/server/blob/blob-s3/pom.xml +++ b/server/blob/blob-s3/pom.xml @@ -105,6 +105,11 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.testcontainers</groupId> <artifactId>testcontainers</artifactId> diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java new file mode 100644 index 0000000000..3c3ff47c97 --- /dev/null +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java @@ -0,0 +1,123 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.blob.objectstorage.aws; + +import static org.apache.james.blob.api.BlobStoreDAOFixture.SHORT_BYTEARRAY; +import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BLOB_ID; +import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME; +import static org.apache.james.blob.objectstorage.aws.DockerAwsS3Container.ACCESS_KEY_ID; +import static org.apache.james.blob.objectstorage.aws.DockerAwsS3Container.SECRET_ACCESS_KEY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URI; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.BlobStoreDAO; +import org.apache.james.blob.api.BlobStoreDAOContract; +import org.apache.james.blob.api.HashBlobId; +import org.apache.james.blob.api.TestBlobId; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import software.amazon.awssdk.services.s3.model.S3Exception; + +@Testcontainers +public class S3MinioTest implements BlobStoreDAOContract { + + private static final int MINIO_PORT = 9000; + private static S3BlobStoreDAO testee; + + @Container + private static final GenericContainer<?> minioContainer = new GenericContainer<>("quay.io/minio/minio") + .withExposedPorts(MINIO_PORT) + .withEnv("MINIO_ROOT_USER", ACCESS_KEY_ID) + .withEnv("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY) + .withCommand("server", "/data", "--console-address", ":9090") + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-minio-s3-test")); + + + @BeforeAll + static void setUp() { + AwsS3AuthConfiguration authConfiguration = AwsS3AuthConfiguration.builder() + .endpoint(URI.create(String.format("http://%s:%s/", minioContainer.getHost(), minioContainer.getMappedPort(MINIO_PORT)))) + .accessKeyId(ACCESS_KEY_ID) + .secretKey(SECRET_ACCESS_KEY) + .build(); + + S3BlobStoreConfiguration s3Configuration = S3BlobStoreConfiguration.builder() + .authConfiguration(authConfiguration) + .region(DockerAwsS3Container.REGION) + .build(); + + testee = new S3BlobStoreDAO(s3Configuration, new TestBlobId.Factory()); + } + + @AfterAll + static void tearDownClass() { + testee.close(); + } + + @AfterEach + void tearDown() { + testee.deleteAllBuckets().block(); + } + + @Override + public BlobStoreDAO testee() { + return testee; + } + + @Test + void saveWillThrowWhenBlobIdHasSlashCharacters() { + BlobId invalidBlobId = new TestBlobId("test-blob//id"); + assertThatThrownBy(() -> Mono.from(testee.save(TEST_BUCKET_NAME, invalidBlobId, SHORT_BYTEARRAY)).block()) + .isInstanceOf(S3Exception.class) + .hasMessageContaining("Object name contains unsupported characters"); + } + + @Test + void saveShouldWorkWhenValidBlobId() { + Mono.from(testee.save(TEST_BUCKET_NAME, TEST_BLOB_ID, SHORT_BYTEARRAY)).block(); + assertThat(Mono.from(testee.readBytes(TEST_BUCKET_NAME, TEST_BLOB_ID)).block()).isEqualTo(SHORT_BYTEARRAY); + } + + @Test + void saveShouldWorkForAllPayloadsWithHash() { + Flux.range(0, 1000) + .concatMap(i -> { + byte[] payload = RandomStringUtils.random(128).getBytes(StandardCharsets.UTF_8); + BlobId blobId = new HashBlobId.Factory().forPayload(payload); + return testee.save(TEST_BUCKET_NAME, blobId, payload); + }) + .then() + .block(); + } + +} diff --git a/src/site/xdoc/server/config-system.xml b/src/site/xdoc/server/config-system.xml index 55650a198a..86b499b03a 100644 --- a/src/site/xdoc/server/config-system.xml +++ b/src/site/xdoc/server/config-system.xml @@ -234,6 +234,9 @@ Should we add the host in the MDC logging context for incoming IMAP, SMTP, POP3? Doing so, a DNS resolution is attempted for each incoming connection, which can be costly. Remote IP is always added to the logging context.</dd> + <dt><strong>james.blob.id.hash.encoding</strong></dt> + <dd>Optional. String. Defaults to base64Url. <br/> + The encoding type when encode blobId. The support value are: base16, hex, base32, base32Hex, base64, base64Url.</dd> </dl> </subsection> --------------------------------------------------------------------- To unsubscribe, e-mail: notifications-unsubscr...@james.apache.org For additional commands, e-mail: notifications-h...@james.apache.org