This is an automated email from the ASF dual-hosted git repository. slachiewicz pushed a commit to branch agent/transport-s3 in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
commit 836f2993f26ae9e49345a746942091ef96c2d16f Author: Sylwester Lachiewicz <[email protected]> AuthorDate: Sat Nov 7 23:48:30 2026 +0100 Add an S3 transport built on the AWS SDK for Java 2.x Step 1 of #2289: the MinIO transporter ported call for call onto the AWS SDK, with the SDK default credential chain as fallback, a region key, path-style access and WHEN_REQUIRED checksums so that S3-compatible stores keep working. The MinIO module is left as it is until this one has shipped. --- maven-resolver-tools/pom.xml | 6 + maven-resolver-transport-s3/pom.xml | 146 +++++++++++++ .../eclipse/aether/transport/s3/ObjectName.java | 82 +++++++ .../aether/transport/s3/ObjectNameMapper.java | 31 +++ .../transport/s3/ObjectNameMapperFactory.java | 41 ++++ .../eclipse/aether/transport/s3/S3Transporter.java | 237 +++++++++++++++++++++ .../s3/S3TransporterConfigurationKeys.java | 89 ++++++++ .../aether/transport/s3/S3TransporterFactory.java | 124 +++++++++++ .../FixedBucketObjectNameMapperFactory.java | 59 +++++ .../RepositoryIdObjectNameMapperFactory.java | 50 +++++ .../eclipse/aether/transport/s3/package-info.java | 34 +++ maven-resolver-transport-s3/src/site/site.xml | 32 +++ .../transport/s3/S3TransporterFactoryTest.java | 115 ++++++++++ .../aether/transport/s3/S3TransporterIT.java | 228 ++++++++++++++++++++ .../FixedBucketObjectNameMapperFactoryTest.java | 66 ++++++ pom.xml | 1 + src/site/markdown/deployment.md | 3 +- 17 files changed, 1343 insertions(+), 1 deletion(-) diff --git a/maven-resolver-tools/pom.xml b/maven-resolver-tools/pom.xml index 219daa80b..bcb9b9cfb 100644 --- a/maven-resolver-tools/pom.xml +++ b/maven-resolver-tools/pom.xml @@ -114,6 +114,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-transport-s3</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.apache.maven.resolver</groupId> <artifactId>maven-resolver-transport-url</artifactId> diff --git a/maven-resolver-transport-s3/pom.xml b/maven-resolver-transport-s3/pom.xml new file mode 100644 index 000000000..7856ddacb --- /dev/null +++ b/maven-resolver-transport-s3/pom.xml @@ -0,0 +1,146 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver</artifactId> + <version>2.0.26-SNAPSHOT</version> + </parent> + + <artifactId>maven-resolver-transport-s3</artifactId> + <packaging>jar</packaging> + + <name>Maven Artifact Resolver Transport S3</name> + <description>Maven Artifact Transport for the S3 API, built on the AWS SDK for Java 2.x.</description> + + <properties> + <version.awssdk>2.57.11</version.awssdk> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>bom</artifactId> + <version>${version.awssdk}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-spi</artifactId> + </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-util</artifactId> + </dependency> + <dependency> + <groupId>javax.inject</groupId> + <artifactId>javax.inject</artifactId> + <scope>provided</scope> + <optional>true</optional> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>s3</artifactId> + <exclusions> + <!-- The SDK's own HTTP engines are not needed: the JDK URL connection client below covers GET, HEAD and PUT --> + <exclusion> + <groupId>software.amazon.awssdk</groupId> + <artifactId>apache-client</artifactId> + </exclusion> + <exclusion> + <groupId>software.amazon.awssdk</groupId> + <artifactId>apache5-client</artifactId> + </exclusion> + <exclusion> + <groupId>software.amazon.awssdk</groupId> + <artifactId>netty-nio-client</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>url-connection-client</artifactId> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <version>${version.testcontainers}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-junit-jupiter</artifactId> + <version>${version.testcontainers}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-minio</artifactId> + <version>${version.testcontainers}</version> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.eclipse.sisu</groupId> + <artifactId>sisu-maven-plugin</artifactId> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>run-its</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectName.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectName.java new file mode 100644 index 000000000..5ba5829fc --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectName.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +/** + * S3 Object name, bucket + name. + * + * @since 2.0.26 + */ +public final class ObjectName { + private final String bucket; + private final String name; + private final int hashCode; + + public ObjectName(String bucket, String name) { + this.bucket = requireNonNull(bucket); + this.name = requireNonNull(name); + + if (bucket.contains("/")) { + throw new IllegalArgumentException("invalid bucket name: " + bucket); + } + if (name.contains("\\")) { + throw new IllegalArgumentException("invalid object name: " + name); + } + + this.hashCode = Objects.hash(bucket, name); + } + + public String getBucket() { + return bucket; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ObjectName that = (ObjectName) o; + return Objects.equals(bucket, that.bucket) && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public String toString() { + return bucket + "/" + name; + } + + public static String normalize(String name) { + return name.replace('\\', '/'); + } +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectNameMapper.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectNameMapper.java new file mode 100644 index 000000000..c5e30111d --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectNameMapper.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +/** + * Mapping for {@link ObjectName}. + * + * @since 2.0.26 + */ +public interface ObjectNameMapper { + /** + * Creates a mapping onto {@link ObjectName}. Should never return {@code null}. + */ + ObjectName name(String path); +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectNameMapperFactory.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectNameMapperFactory.java new file mode 100644 index 000000000..2a6e3c283 --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/ObjectNameMapperFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import java.util.Map; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * Factory for {@link ObjectNameMapper}. + * + * @since 2.0.26 + */ +public interface ObjectNameMapperFactory { + /** + * Creates a factory for mapping artifact layout path onto {@link ObjectName}. Should never return {@code null}. + */ + ObjectNameMapper create( + RepositorySystemSession session, + RemoteRepository repository, + S3Client s3Client, + Map<String, String> extraHeaders); +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3Transporter.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3Transporter.java new file mode 100644 index 000000000..f68ec5ef0 --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3Transporter.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.AuthenticationContext; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.AbstractTransporter; +import org.eclipse.aether.spi.connector.transport.GetTask; +import org.eclipse.aether.spi.connector.transport.PeekTask; +import org.eclipse.aether.spi.connector.transport.PutTask; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.spi.io.PathProcessor; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.util.ConfigUtils; +import org.eclipse.aether.util.connector.transport.http.HttpTransporterUtils; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import static java.util.Objects.requireNonNull; + +/** + * A transporter for the S3 API, built on the AWS SDK for Java 2.x. + * + * @since 2.0.26 + */ +final class S3Transporter extends AbstractTransporter implements Transporter { + private final URI baseUri; + + private final Map<String, String> headers; + + private final S3Client client; + + private final ObjectNameMapper objectNameMapper; + + private final PathProcessor pathProcessor; + + S3Transporter( + RepositorySystemSession session, + RemoteRepository repository, + ObjectNameMapperFactory objectNameMapperFactory, + PathProcessor pathProcessor) + throws NoTransporterException { + URI endpoint; + try { + URI uri = new URI(repository.getUrl()).parseServerAuthority(); + if (uri.isOpaque()) { + throw new URISyntaxException(repository.getUrl(), "URL must not be opaque"); + } + if (uri.getRawFragment() != null || uri.getRawQuery() != null) { + throw new URISyntaxException(repository.getUrl(), "URL must not have fragment or query"); + } + String path = uri.getPath(); + if (path == null) { + path = "/"; + } + if (!path.startsWith("/")) { + path = "/" + path; + } + if (!path.endsWith("/")) { + path = path + "/"; + } + this.baseUri = URI.create(uri.getScheme() + "://" + uri.getRawAuthority() + path); + // the bucket goes into the path (path-style access), so the endpoint is the authority alone + endpoint = URI.create(uri.getScheme() + "://" + uri.getRawAuthority()); + } catch (URISyntaxException e) { + throw new NoTransporterException(repository, e.getMessage(), e); + } + + HashMap<String, String> headers = new HashMap<>(); + Map<String, String> configuredHeaders = HttpTransporterUtils.getHttpHeaders(session, repository); + if (configuredHeaders != null) { + headers.putAll(configuredHeaders); + } + this.headers = headers; + + this.client = S3Client.builder() + .endpointOverride(endpoint) + .region(region(session, repository)) + .credentialsProvider(credentialsProvider(session, repository)) + .httpClientBuilder(UrlConnectionHttpClient.builder()) + // S3-compatible stores are addressed by path, and most of them do not implement the CRC-based + // integrity protections the SDK sends by default since 2.30.0 + .forcePathStyle(true) + .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) + .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED) + .overrideConfiguration(configuration -> headers.forEach(configuration::putHeader)) + .build(); + this.objectNameMapper = objectNameMapperFactory.create(session, repository, client, headers); + this.pathProcessor = requireNonNull(pathProcessor); + } + + /** + * Repository authentication (username as access key, password as secret key) wins; without it the SDK default + * chain applies, so environment variables, profiles, container and instance roles work as with any AWS tool. + */ + private static AwsCredentialsProvider credentialsProvider( + RepositorySystemSession session, RemoteRepository repository) { + try (AuthenticationContext repoAuthContext = AuthenticationContext.forRepository(session, repository)) { + if (repoAuthContext != null) { + String username = repoAuthContext.get(AuthenticationContext.USERNAME); + String password = repoAuthContext.get(AuthenticationContext.PASSWORD); + if (username != null && password != null) { + return StaticCredentialsProvider.create(AwsBasicCredentials.create(username, password)); + } + } + } + return DefaultCredentialsProvider.builder().build(); + } + + private static Region region(RepositorySystemSession session, RemoteRepository repository) { + String configured = ConfigUtils.getString( + session, + null, + S3TransporterConfigurationKeys.CONFIG_PROP_REGION + "." + repository.getId(), + S3TransporterConfigurationKeys.CONFIG_PROP_REGION); + if (configured != null) { + return Region.of(configured); + } + try { + return DefaultAwsRegionProviderChain.builder().build().getRegion(); + } catch (SdkClientException e) { + return Region.of(S3TransporterConfigurationKeys.DEFAULT_REGION); + } + } + + @Override + public int classify(Throwable error) { + if (error instanceof NoSuchKeyException || error instanceof NoSuchBucketException) { + return ERROR_NOT_FOUND; + } + if (error instanceof S3Exception && ((S3Exception) error).statusCode() == 404) { + return ERROR_NOT_FOUND; + } + return ERROR_OTHER; + } + + @Override + protected void implPeek(PeekTask task) throws Exception { + ObjectName objectName = + objectNameMapper.name(baseUri.relativize(task.getLocation()).getPath()); + client.headObject(HeadObjectRequest.builder() + .bucket(objectName.getBucket()) + .key(objectName.getName()) + .build()); + } + + @Override + protected void implGet(GetTask task) throws Exception { + ObjectName objectName = + objectNameMapper.name(baseUri.relativize(task.getLocation()).getPath()); + try (ResponseInputStream<GetObjectResponse> stream = client.getObject(GetObjectRequest.builder() + .bucket(objectName.getBucket()) + .key(objectName.getName()) + .build())) { + Long contentLength = stream.response().contentLength(); + long length = contentLength != null ? contentLength : -1L; + final Path dataFile = task.getDataPath(); + if (dataFile == null) { + utilGet(task, stream, true, length, false); + } else { + try (PathProcessor.CollocatedTempFile tempFile = pathProcessor.newTempFile(dataFile)) { + task.setDataPath(tempFile.getPath(), false); + utilGet(task, stream, true, length, false); + tempFile.move(); + } finally { + task.setDataPath(dataFile); + } + } + } + } + + @Override + protected void implPut(PutTask task) throws Exception { + ObjectName objectName = + objectNameMapper.name(baseUri.relativize(task.getLocation()).getPath()); + task.getListener().transportStarted(0, task.getDataLength()); + PutObjectRequest request = PutObjectRequest.builder() + .bucket(objectName.getBucket()) + .key(objectName.getName()) + .build(); + final Path dataFile = task.getDataPath(); + if (dataFile == null) { + try (InputStream inputStream = task.newInputStream()) { + client.putObject(request, RequestBody.fromInputStream(inputStream, task.getDataLength())); + } + } else { + client.putObject(request, RequestBody.fromFile(dataFile)); + } + } + + @Override + protected void implClose() { + client.close(); + } +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3TransporterConfigurationKeys.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3TransporterConfigurationKeys.java new file mode 100644 index 000000000..4d77625b0 --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3TransporterConfigurationKeys.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import org.eclipse.aether.ConfigurationProperties; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.transport.s3.internal.FixedBucketObjectNameMapperFactory; + +/** + * Configuration for the S3 transport. + * + * @since 2.0.26 + */ +public final class S3TransporterConfigurationKeys { + private S3TransporterConfigurationKeys() {} + + static final String CONFIG_PROPS_PREFIX = + ConfigurationProperties.PREFIX_TRANSPORT + S3TransporterFactory.NAME + "."; + + /** + * Object name mapper to use. + * + * @configurationSource {@link RepositorySystemSession#getConfigProperties()} + * @configurationType {@link String} + * @configurationDefaultValue {@link #DEFAULT_OBJECT_NAME_MAPPER} + * @configurationRepoIdSuffix Yes + */ + public static final String CONFIG_PROP_OBJECT_NAME_MAPPER = CONFIG_PROPS_PREFIX + "objectNameMapper"; + + public static final String DEFAULT_OBJECT_NAME_MAPPER = FixedBucketObjectNameMapperFactory.NAME; + + /** + * The fixed bucket name to use. + * + * @configurationSource {@link RepositorySystemSession#getConfigProperties()} + * @configurationType {@link String} + * @configurationDefaultValue {@link #DEFAULT_FIXED_BUCKET_NAME} + * @configurationRepoIdSuffix Yes + */ + public static final String CONFIG_PROP_FIXED_BUCKET_NAME = CONFIG_PROPS_PREFIX + "fixedBucketName"; + + public static final String DEFAULT_FIXED_BUCKET_NAME = "maven"; + + /** + * The AWS region the requests are signed for. S3-compatible stores accept any value, but Signature Version 4 + * needs one. When unset, the SDK's default region provider chain is consulted (environment, system properties, + * profile), and {@link #DEFAULT_REGION} is used if that chain yields nothing. + * + * @configurationSource {@link RepositorySystemSession#getConfigProperties()} + * @configurationType {@link String} + * @configurationDefaultValue {@link #DEFAULT_REGION} + * @configurationRepoIdSuffix Yes + */ + public static final String CONFIG_PROP_REGION = CONFIG_PROPS_PREFIX + "region"; + + public static final String DEFAULT_REGION = "us-east-1"; + + /** + * Whether plaintext HTTP object storage endpoints (repository URLs in {@code s3+http} form) are allowed. + * Plaintext endpoints offer no transport integrity (artifacts and their checksum objects travel over the same + * tamperable channel) and, because the repository protocol string is not {@code http}, such URLs are not matched + * by {@code external:http:*} mirror blocking. They are therefore refused unless this property is explicitly set + * to {@code true}. + * + * @configurationSource {@link RepositorySystemSession#getConfigProperties()} + * @configurationType {@link Boolean} + * @configurationDefaultValue {@link #DEFAULT_ALLOW_INSECURE_PROTOCOL} + * @configurationRepoIdSuffix Yes + */ + public static final String CONFIG_PROP_ALLOW_INSECURE_PROTOCOL = CONFIG_PROPS_PREFIX + "allowInsecureProtocol"; + + public static final boolean DEFAULT_ALLOW_INSECURE_PROTOCOL = false; +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3TransporterFactory.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3TransporterFactory.java new file mode 100644 index 000000000..b39897407 --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/S3TransporterFactory.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import javax.inject.Inject; +import javax.inject.Named; + +import java.util.Map; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.spi.connector.transport.TransporterFactory; +import org.eclipse.aether.spi.io.PathProcessor; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.util.ConfigUtils; + +import static java.util.Objects.requireNonNull; + +/** + * A transporter factory for repositories served over the S3 API, built on the AWS SDK for Java 2.x. + * + * @since 2.0.26 + */ +@Named(S3TransporterFactory.NAME) +public final class S3TransporterFactory implements TransporterFactory { + public static final String NAME = "s3"; + + /** + * Above the MinIO transporter's default, so that an {@code s3+https} repository resolves to this transporter + * when both modules are present. + */ + private static final float DEFAULT_PRIORITY = 1.0f; + + private float priority = DEFAULT_PRIORITY; + + private final Map<String, ObjectNameMapperFactory> objectNameMapperFactories; + + private final PathProcessor pathProcessor; + + @Inject + public S3TransporterFactory( + Map<String, ObjectNameMapperFactory> objectNameMapperFactories, PathProcessor pathProcessor) { + this.objectNameMapperFactories = requireNonNull(objectNameMapperFactories, "objectNameMapperFactories"); + this.pathProcessor = requireNonNull(pathProcessor, "pathProcessor"); + } + + @Override + public float getPriority() { + return priority; + } + + public S3TransporterFactory setPriority(float priority) { + this.priority = priority; + return this; + } + + @Override + public Transporter newInstance(RepositorySystemSession session, RemoteRepository repository) + throws NoTransporterException { + requireNonNull(session, "session cannot be null"); + requireNonNull(repository, "repository cannot be null"); + // "s3+http(s)" is the opt-in form; plain HTTP(S) repositories are only taken over when the user raised the + // priority of this transporter on purpose. + RemoteRepository adjusted = repository; + boolean prefixStripped = false; + if ("s3+http".equalsIgnoreCase(repository.getProtocol()) + || "s3+https".equalsIgnoreCase(repository.getProtocol())) { + adjusted = new RemoteRepository.Builder(repository) + .setUrl(repository.getUrl().substring("s3+".length())) + .build(); + prefixStripped = true; + } else if (priority == DEFAULT_PRIORITY) { + throw new NoTransporterException( + repository, "To use the S3 transport with plain HTTP/HTTPS repositories, increase its priority"); + } + // Plaintext endpoints from "s3+http" URLs are not matched by "external:http:*" mirror blocking (the + // repository protocol string is not "http") and offer no transport integrity: artifacts and their checksum + // objects travel over the same tamperable channel. Refuse them unless the operator explicitly opted in. + if (prefixStripped && "http".equalsIgnoreCase(adjusted.getProtocol())) { + boolean allowInsecureProtocol = ConfigUtils.getBoolean( + session, + S3TransporterConfigurationKeys.DEFAULT_ALLOW_INSECURE_PROTOCOL, + S3TransporterConfigurationKeys.CONFIG_PROP_ALLOW_INSECURE_PROTOCOL + "." + repository.getId(), + S3TransporterConfigurationKeys.CONFIG_PROP_ALLOW_INSECURE_PROTOCOL); + if (!allowInsecureProtocol) { + throw new NoTransporterException( + repository, + "Plaintext HTTP object storage endpoints are insecure (no transport integrity; not subject" + + " to 'external:http:*' mirror blocking) and are refused by default: use the" + + " 's3+https' URL form, or explicitly opt in by setting the configuration property '" + + S3TransporterConfigurationKeys.CONFIG_PROP_ALLOW_INSECURE_PROTOCOL + + "' (optionally suffixed with '." + repository.getId() + "') to 'true'"); + } + } + String objectNameMapperConf = ConfigUtils.getString( + session, + S3TransporterConfigurationKeys.DEFAULT_OBJECT_NAME_MAPPER, + S3TransporterConfigurationKeys.CONFIG_PROP_OBJECT_NAME_MAPPER + "." + repository.getId(), + S3TransporterConfigurationKeys.CONFIG_PROP_OBJECT_NAME_MAPPER); + ObjectNameMapperFactory objectNameMapperFactory = objectNameMapperFactories.get(objectNameMapperConf); + if (objectNameMapperFactory == null) { + throw new IllegalArgumentException("Unknown object name mapper configured '" + objectNameMapperConf + + "' for repository " + repository.getId()); + } + return new S3Transporter(session, adjusted, objectNameMapperFactory, pathProcessor); + } +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/internal/FixedBucketObjectNameMapperFactory.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/internal/FixedBucketObjectNameMapperFactory.java new file mode 100644 index 000000000..f1f630cad --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/internal/FixedBucketObjectNameMapperFactory.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3.internal; + +import javax.inject.Named; +import javax.inject.Singleton; + +import java.util.Map; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.transport.s3.ObjectName; +import org.eclipse.aether.transport.s3.ObjectNameMapper; +import org.eclipse.aether.transport.s3.ObjectNameMapperFactory; +import org.eclipse.aether.transport.s3.S3TransporterConfigurationKeys; +import org.eclipse.aether.util.ConfigUtils; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * A fixed bucket mapper, uses given bucket ID and then constructs object name using repository ID and layout path as + * object name. + */ +@Singleton +@Named(FixedBucketObjectNameMapperFactory.NAME) +public class FixedBucketObjectNameMapperFactory implements ObjectNameMapperFactory { + public static final String NAME = "fixedBucket"; + + @Override + public ObjectNameMapper create( + RepositorySystemSession session, + RemoteRepository repository, + S3Client unused, + Map<String, String> headers) { + // ConfigUtils.getString(session, defaultValue, keys...): the default value ("maven") comes first, + // then the repository-suffixed key, then the global key. + String bucket = ConfigUtils.getString( + session, + S3TransporterConfigurationKeys.DEFAULT_FIXED_BUCKET_NAME, + S3TransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME + "." + repository.getId(), + S3TransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME); + return path -> new ObjectName(bucket, repository.getId() + "/" + ObjectName.normalize(path)); + } +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/internal/RepositoryIdObjectNameMapperFactory.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/internal/RepositoryIdObjectNameMapperFactory.java new file mode 100644 index 000000000..e8795f654 --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/internal/RepositoryIdObjectNameMapperFactory.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3.internal; + +import javax.inject.Named; +import javax.inject.Singleton; + +import java.util.Map; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.transport.s3.ObjectName; +import org.eclipse.aether.transport.s3.ObjectNameMapper; +import org.eclipse.aether.transport.s3.ObjectNameMapperFactory; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * A mapper that uses repository ID as bucket name and layout path as object name. Assumes repository ID is a valid + * bucket name. + */ +@Singleton +@Named(RepositoryIdObjectNameMapperFactory.NAME) +public class RepositoryIdObjectNameMapperFactory implements ObjectNameMapperFactory { + public static final String NAME = "repositoryId"; + + @Override + public ObjectNameMapper create( + RepositorySystemSession session, + RemoteRepository repository, + S3Client unused, + Map<String, String> headers) { + return path -> new ObjectName(repository.getId(), ObjectName.normalize(path)); + } +} diff --git a/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/package-info.java b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/package-info.java new file mode 100644 index 000000000..31a7a42e6 --- /dev/null +++ b/maven-resolver-transport-s3/src/main/java/org/eclipse/aether/transport/s3/package-info.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Support for downloads and uploads over the S3 API. The implementation is built on the + * <a href="https://github.com/aws/aws-sdk-java-v2">AWS SDK for Java 2.x</a> with its JDK URL connection HTTP client, + * uses path-style access, and does not send the CRC-based request checksums that S3-compatible stores commonly lack. + * The repository URL should be defined with protocol {@code s3+https}; the factory merely strips the "s3+" prefix + * assuming the resulting URL points to the S3 endpoint, with the bucket chosen by the configured object name mapper. + * Credentials come from the repository authentication (username as access key, password as secret key) when set, + * and from the SDK default credentials provider chain otherwise. + * The plaintext {@code s3+http} form is refused by default: it offers no transport integrity and is not matched by + * {@code external:http:*} mirror blocking. To use a plaintext endpoint anyway, explicitly set the configuration + * property {@code aether.transport.s3.allowInsecureProtocol} to {@code true}. + * + * @since 2.0.26 + */ +package org.eclipse.aether.transport.s3; diff --git a/maven-resolver-transport-s3/src/site/site.xml b/maven-resolver-transport-s3/src/site/site.xml new file mode 100644 index 000000000..2101ed517 --- /dev/null +++ b/maven-resolver-transport-s3/src/site/site.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<site xmlns="http://maven.apache.org/SITE/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SITE/2.0.0 https://maven.apache.org/xsd/site-2.0.0.xsd" name="Transport S3"> + <body> + <menu name="Overview"> + <item name="Introduction" href="index.html"/> + <item name="JavaDocs" href="apidocs/index.html"/> + <item name="Source Xref" href="xref/index.html"/> + </menu> + <menu ref="parent"/> + <menu ref="reports"/> + </body> +</site> diff --git a/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/S3TransporterFactoryTest.java b/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/S3TransporterFactoryTest.java new file mode 100644 index 000000000..f7d636099 --- /dev/null +++ b/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/S3TransporterFactoryTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.spi.io.PathProcessorSupport; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.transport.s3.internal.FixedBucketObjectNameMapperFactory; +import org.eclipse.aether.transport.s3.internal.RepositoryIdObjectNameMapperFactory; +import org.eclipse.aether.util.repository.AuthenticationBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +class S3TransporterFactoryTest { + private S3TransporterFactory factory; + + private RepositorySystemSession session; + + private RemoteRepository repository; + + @BeforeEach + void startSuite() throws Exception { + Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir"))); // hack for Surefire + Map<String, ObjectNameMapperFactory> factories = new HashMap<>(); + factories.put(RepositoryIdObjectNameMapperFactory.NAME, new RepositoryIdObjectNameMapperFactory()); + factories.put(FixedBucketObjectNameMapperFactory.NAME, new FixedBucketObjectNameMapperFactory()); + factory = new S3TransporterFactory(factories, new PathProcessorSupport()); + session = new DefaultRepositorySystemSession(h -> true); + repository = withAuth(new RemoteRepository.Builder("repo", "default", "s3+https://localhost")) + .build(); + } + + private static RemoteRepository.Builder withAuth(RemoteRepository.Builder builder) { + return builder.setAuthentication(new AuthenticationBuilder() + .addUsername("username") + .addPassword("password") + .build()); + } + + @Test + void goodUrl() throws Exception { + try (Transporter transporter = factory.newInstance(session, repository)) { + // nope + } + } + + @Test + void goodUrlWithoutRepositoryAuthFallsBackToTheSdkChain() throws Exception { + RemoteRepository anonymous = new RemoteRepository.Builder("repo", "default", "s3+https://localhost").build(); + try (Transporter transporter = factory.newInstance(session, anonymous)) { + // credentials are resolved per request, so construction must not fail + } + } + + @Test + void plainHttpUrlsAreNotTakenOverByDefault() { + for (String url : new String[] {"http://localhost", "https://localhost"}) { + RemoteRepository plain = withAuth(new RemoteRepository.Builder("repo", "default", url)) + .build(); + assertThrows(NoTransporterException.class, () -> factory.newInstance(session, plain)); + } + } + + @Test + void plaintextHttpRefusedByDefault() { + RemoteRepository plaintext = withAuth(new RemoteRepository.Builder("repo", "default", "s3+http://localhost")) + .build(); + assertThrows(NoTransporterException.class, () -> factory.newInstance(session, plaintext)); + } + + @Test + void plaintextHttpAllowedWithExplicitOptIn() throws Exception { + DefaultRepositorySystemSession insecureSession = new DefaultRepositorySystemSession(h -> true); + insecureSession.setConfigProperty( + S3TransporterConfigurationKeys.CONFIG_PROP_ALLOW_INSECURE_PROTOCOL, Boolean.TRUE); + RemoteRepository plaintext = withAuth(new RemoteRepository.Builder("repo", "default", "s3+http://localhost")) + .build(); + try (Transporter transporter = factory.newInstance(insecureSession, plaintext)) { + // nope + } + } + + @Test + void unknownObjectNameMapperIsRejected() { + DefaultRepositorySystemSession misconfigured = new DefaultRepositorySystemSession(h -> true); + misconfigured.setConfigProperty(S3TransporterConfigurationKeys.CONFIG_PROP_OBJECT_NAME_MAPPER, "nope"); + assertThrows(IllegalArgumentException.class, () -> factory.newInstance(misconfigured, repository)); + } +} diff --git a/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/S3TransporterIT.java b/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/S3TransporterIT.java new file mode 100644 index 000000000..adaba20d9 --- /dev/null +++ b/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/S3TransporterIT.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.GetTask; +import org.eclipse.aether.spi.connector.transport.PeekTask; +import org.eclipse.aether.spi.connector.transport.PutTask; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.spi.io.PathProcessor; +import org.eclipse.aether.spi.io.PathProcessorSupport; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.transport.s3.internal.RepositoryIdObjectNameMapperFactory; +import org.eclipse.aether.util.repository.AuthenticationBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * S3 transporter IT against an S3-compatible store in a container. + */ +@Testcontainers(disabledWithoutDocker = true) +class S3TransporterIT { + private static final String BUCKET_NAME = "s3-repo"; + + private static final String OBJECT_NAME = "dir/file.txt"; + + private static final String OBJECT_CONTENT = "content"; + + private PathProcessor pathProcessor; + + private MinIOContainer container; + + private RepositorySystemSession session; + + private ObjectNameMapperFactory objectNameMapperFactory; + + @BeforeEach + void startSuite() throws Exception { + Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir"))); // hack for Surefire + pathProcessor = new PathProcessorSupport(); + container = new MinIOContainer(DockerImageName.parse("pgsty/silo:RELEASE.2026-09-03T13-18-01Z-distroless") + .asCompatibleSubstituteFor("minio/minio")); + container.start(); + try (S3Client s3 = S3Client.builder() + .endpointOverride(URI.create(container.getS3URL())) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(container.getUserName(), container.getPassword()))) + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .forcePathStyle(true) + .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) + .build()) { + s3.createBucket(b -> b.bucket(BUCKET_NAME)); + s3.putObject(b -> b.bucket(BUCKET_NAME).key(OBJECT_NAME), RequestBody.fromString(OBJECT_CONTENT)); + } + session = new DefaultRepositorySystemSession(h -> true); + objectNameMapperFactory = new RepositoryIdObjectNameMapperFactory(); + } + + @AfterEach + void stopSuite() { + container.stop(); + } + + enum RepositoryAuth { + WITHOUT, + WITH, + WRONG + } + + private RemoteRepository newRepo(RepositoryAuth auth) { + RemoteRepository.Builder builder = new RemoteRepository.Builder(BUCKET_NAME, "default", container.getS3URL()); + if (auth == RepositoryAuth.WITH) { + builder.setAuthentication(new AuthenticationBuilder() + .addUsername(container.getUserName()) + .addPassword(container.getPassword()) + .build()); + } else if (auth == RepositoryAuth.WRONG) { + builder.setAuthentication(new AuthenticationBuilder() + .addUsername("wrongusername") + .addPassword("wrongpassword") + .build()); + } + return builder.build(); + } + + private S3Transporter newTransporter(RepositoryAuth auth) throws NoTransporterException { + return new S3Transporter(session, newRepo(auth), objectNameMapperFactory, pathProcessor); + } + + @Test + void peekWithoutAuth() throws NoTransporterException { + // no repository auth and (normally) nothing in the SDK chain: the failure surfaces per request, as ERROR_OTHER + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITHOUT)) { + Exception e = assertThrows(Exception.class, () -> transporter.peek(new PeekTask(URI.create("test")))); + assertEquals(Transporter.ERROR_OTHER, transporter.classify(e)); + } + } + + @Test + void peekWithWrongAuth() throws NoTransporterException { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WRONG)) { + Exception e = assertThrows(Exception.class, () -> transporter.peek(new PeekTask(URI.create("test")))); + assertInstanceOf(S3Exception.class, e); + assertEquals(Transporter.ERROR_OTHER, transporter.classify(e)); + } + } + + @Test + void peekNonexistent() throws NoTransporterException { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + Exception e = assertThrows(Exception.class, () -> transporter.peek(new PeekTask(URI.create("test")))); + assertEquals(Transporter.ERROR_NOT_FOUND, transporter.classify(e)); + } + } + + @Test + void peekExistent() throws Exception { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + transporter.peek(new PeekTask(URI.create(OBJECT_NAME))); + } + } + + @Test + void getNonexistent() throws NoTransporterException { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + Exception e = assertThrows(Exception.class, () -> transporter.get(new GetTask(URI.create("test")))); + assertInstanceOf(NoSuchKeyException.class, e); + assertEquals(Transporter.ERROR_NOT_FOUND, transporter.classify(e)); + } + } + + @Test + void getExistent() throws Exception { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + GetTask task = new GetTask(URI.create(OBJECT_NAME)); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void getExistentToFile() throws Exception { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + java.nio.file.Path target = Files.createTempDirectory("s3-it").resolve("file.txt"); + GetTask task = new GetTask(URI.create(OBJECT_NAME)).setDataPath(target); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(Files.readAllBytes(target), StandardCharsets.UTF_8)); + } + } + + @Test + void putNonexistent() throws Exception { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + URI uri = URI.create("test"); + transporter.put(new PutTask(uri).setDataBytes(OBJECT_CONTENT.getBytes(StandardCharsets.UTF_8))); + GetTask task = new GetTask(uri); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void putExistent() throws Exception { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + URI uri = URI.create(OBJECT_NAME); + GetTask task = new GetTask(uri); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + String altContent = "altContent"; + transporter.put(new PutTask(uri).setDataBytes(altContent.getBytes(StandardCharsets.UTF_8))); + task = new GetTask(uri); + transporter.get(task); + assertEquals(altContent, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void putFromFile() throws Exception { + try (S3Transporter transporter = newTransporter(RepositoryAuth.WITH)) { + java.nio.file.Path source = Files.createTempDirectory("s3-it").resolve("source.txt"); + Files.write(source, "fromFile".getBytes(StandardCharsets.UTF_8)); + URI uri = URI.create("dir/fromFile.txt"); + transporter.put(new PutTask(uri).setDataPath(source)); + GetTask task = new GetTask(uri); + transporter.get(task); + assertEquals("fromFile", new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } + } +} diff --git a/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/internal/FixedBucketObjectNameMapperFactoryTest.java b/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/internal/FixedBucketObjectNameMapperFactoryTest.java new file mode 100644 index 000000000..422bbad18 --- /dev/null +++ b/maven-resolver-transport-s3/src/test/java/org/eclipse/aether/transport/s3/internal/FixedBucketObjectNameMapperFactoryTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.s3.internal; + +import java.util.Collections; + +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.transport.s3.ObjectNameMapper; +import org.eclipse.aether.transport.s3.S3TransporterConfigurationKeys; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * UT asserting the effective bucket is exactly the one the operator configured, with the documented default. + */ +class FixedBucketObjectNameMapperFactoryTest { + private final FixedBucketObjectNameMapperFactory factory = new FixedBucketObjectNameMapperFactory(); + + private final RemoteRepository repository = + new RemoteRepository.Builder("repo", "default", "s3+https://localhost").build(); + + private String effectiveBucket(DefaultRepositorySystemSession session) { + ObjectNameMapper mapper = factory.create(session, repository, null, Collections.emptyMap()); + return mapper.name("g/a/v/a.jar").getBucket(); + } + + @Test + void documentedDefaultBucketNameIsUsed() { + DefaultRepositorySystemSession session = new DefaultRepositorySystemSession(h -> true); + assertEquals(S3TransporterConfigurationKeys.DEFAULT_FIXED_BUCKET_NAME, effectiveBucket(session)); + } + + @Test + void globalConfigurationKeyIsHonored() { + DefaultRepositorySystemSession session = new DefaultRepositorySystemSession(h -> true); + session.setConfigProperty(S3TransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME, "custom-bucket"); + assertEquals("custom-bucket", effectiveBucket(session)); + } + + @Test + void repositoryIdSuffixedKeyTakesPrecedence() { + DefaultRepositorySystemSession session = new DefaultRepositorySystemSession(h -> true); + session.setConfigProperty(S3TransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME, "custom-bucket"); + session.setConfigProperty( + S3TransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME + "." + repository.getId(), "repo-bucket"); + assertEquals("repo-bucket", effectiveBucket(session)); + } +} diff --git a/pom.xml b/pom.xml index 470838fc4..b188ad6a0 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ <module>maven-resolver-transport-jdk-parent</module> <module>maven-resolver-transport-jetty</module> <module>maven-resolver-transport-minio</module> + <module>maven-resolver-transport-s3</module> <module>maven-resolver-transport-url</module> <module>maven-resolver-transport-wagon</module> <module>maven-resolver-generator-gnupg</module> diff --git a/src/site/markdown/deployment.md b/src/site/markdown/deployment.md index 4fee706be..df754f499 100644 --- a/src/site/markdown/deployment.md +++ b/src/site/markdown/deployment.md @@ -47,5 +47,6 @@ URL Protocol | Implementation | Description `file`, `bundle` | `org.eclipse.aether.transport.file.FileTransporter` | Writes the artifact or metadata to the file system. `http`, `https` | multiple | Issues an HTTP PUT request for each artifact or metadata. `classpath` | `org.eclipse.aether.transport.classpath.ClasspathTransporter` | Unsupported -`minio+http`, `minio+https`, `s3+http`, `s3+https` | `org.eclipse.aether.transport.minio.MinioTransporter` | Uploads the artifact or metadata as an object to the bucket. The configuration defines how the transporter converts the location from the `RepositoryLayout` into an object name and a bucket name. +`s3+http`, `s3+https` | `org.eclipse.aether.transport.s3.S3Transporter` | Uploads the artifact or metadata as an object to the bucket through the AWS SDK for Java 2.x. The configuration defines how the transporter converts the location from the `RepositoryLayout` into an object name and a bucket name. +`minio+http`, `minio+https` | `org.eclipse.aether.transport.minio.MinioTransporter` | Uploads the artifact or metadata as an object to the bucket. The configuration defines how the transporter converts the location from the `RepositoryLayout` into an object name and a bucket name. `*` | `org.eclipse.aether.transport.wagon.WagonTransporter` | Calls `StreamingWagon.putFromStream(...)` or `Wagon.put(...)`. [Apache Wagon](https://maven.apache.org/wagon/) provides further details.
