rishabhdaim commented on code in PR #2558:
URL: https://github.com/apache/jackrabbit-oak/pull/2558#discussion_r2416520496
##########
oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java:
##########
@@ -216,54 +290,384 @@ public static Properties readConfig(String fileName)
throws IOException {
return prop;
}
- private static void deleteIfPossible(final File file) {
- boolean deleted = file.delete();
- if (!deleted) {
- LOG.warn("Could not delete " + file.getAbsolutePath());
+ public static Map<String, Object> asMap(Properties props) {
+ Map<String, Object> map = new HashMap<>();
+ for (Object key : props.keySet()) {
+ map.put((String)key, props.get(key));
}
+ return map;
}
- private static ClientConfiguration getClientConfiguration(Properties prop)
{
- int connectionTimeOut =
Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT));
- int socketTimeOut =
Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT));
- int maxConnections =
Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS));
+ /**
+ * Determines the data encryption type to use for S3 operations based on
the provided properties.
+ * <p>
+ * If the property {@code S3_ENCRYPTION} is set, returns the corresponding
{@link DataEncryption} enum value.
+ * If the property is not set or is invalid, returns {@link
DataEncryption#NONE}.
+ *
+ * @param props the properties containing S3 configuration
+ * @return the {@link DataEncryption} type to use
+ */
+ public static DataEncryption getDataEncryption(final Properties props) {
+ final String encryptionType = props.getProperty(S3_ENCRYPTION);
+ DataEncryption encryption = DataEncryption.NONE;
+ if (encryptionType != null) {
+ encryption = DataEncryption.valueOf(encryptionType);
+ }
+ return encryption;
+ }
+
+ /**
+ * Determines the AWS region to use based on the provided properties.
+ * <p>
+ * The method attempts to extract the region in the following order:
+ * <ol>
+ * <li>If the S3 endpoint property is set, tries to parse the region
from the endpoint URL.</li>
+ * <li>If not found, uses the S3 region property.</li>
+ * <li>If the region property is empty, falls back to the default region
from the environment.</li>
+ * <li>If the region is "us-standard", returns "us-east-1".</li>
+ * </ol>
+ *
+ * @param prop the properties containing S3 configuration
+ * @return the AWS region as a string
+ */
+ static String getRegion(final Properties prop) {
+
+ String region = null;
+
+ if (Objects.nonNull(prop.getProperty(S3Constants.S3_END_POINT))) {
+ region =
getRegionFromEndpoint(prop.getProperty(S3Constants.S3_END_POINT));
+ }
+
+ if (Objects.nonNull(region)) {
+ return region;
+ }
+
+ region = prop.getProperty(S3Constants.S3_REGION);
+
+ if (region.isEmpty()) {
+ region = Utils.getDefaultRegion();
+ }
+
+ if (Objects.equals(DEFAULT_AWS_BUCKET_REGION, region)) {
+ return US_EAST_1_AWS_BUCKET_REGION;
+ }
+ return region;
+ }
+
+ /**
+ * Extracts the AWS region from a given S3 endpoint URL.
+ * <p>
+ * Supports both path-style and virtual-hosted-style S3 endpoints, e.g.:
+ * <ul>
+ * <li>https://s3.eu-west-1.amazonaws.com</li>
+ * <li>https://bucket.s3.eu-west-1.amazonaws.com</li>
+ * <li>https://s3.amazonaws.com (returns us-east-1)</li>
+ * </ul>
+ * If the region cannot be determined, returns null.
+ *
+ * @param endpoint the S3 endpoint URL as a string
+ * @return the AWS region string, or null if not found
+ */
+ static String getRegionFromEndpoint(final String endpoint) {
+ try {
+ URI uri = URI.create(endpoint);
+ String host = uri.getHost();
+
+ // Pattern for standard S3 endpoints: s3.region.amazonaws.com or
bucket.s3.region.amazonaws.com
+ Pattern pattern =
Pattern.compile("s3[.-]([a-z0-9-]+)\\.amazonaws\\.com");
+ Matcher matcher = pattern.matcher(host);
+
+ if (matcher.find()) {
+ return matcher.group(1);
+ }
+
+ // Handle us-east-1 special case (s3.amazonaws.com)
+ // Handle virtual-hosted-style URLs: bucket.s3.amazonaws.com
+ if (host.equals("s3.amazonaws.com") ||
host.endsWith(".s3.amazonaws.com")) {
+ return Region.US_EAST_1.id();
+ }
+
+ LOG.warn("Cannot parse region from endpoint: {}", endpoint);
+
+ return null;
+
+ } catch (Exception e) {
+ LOG.error("Invalid endpoint format: {}", endpoint);
+ return null;
+ }
+ }
+
+ static void deleteBucketObjects(final String bucket, final Properties
props, final S3Client s3service,
+ final ListObjectsV2Iterable
listResponses) {
+ for (ListObjectsV2Response listRes : listResponses) {
+ List<ObjectIdentifier> deleteList = new ArrayList<>();
+ List<String> keysToDelete = new ArrayList<>();
+ for (S3Object s3Obj : listRes.contents()) {
+
deleteList.add(ObjectIdentifier.builder().key(s3Obj.key()).build());
+ keysToDelete.add(s3Obj.key());
+ }
+
+ if (!deleteList.isEmpty()) {
+ RemoteStorageMode mode = getMode(props);
+ if (mode == RemoteStorageMode.S3) {
+ s3service.deleteObjects(delReq ->
+ delReq.bucket(bucket)
+ .delete(delObj ->
+ delObj.objects(deleteList)
+ .build())
+ .build());
+ } else {
+ // Delete objects one by one
+ keysToDelete.forEach(key -> s3service.deleteObject(delObj
-> delObj
+ .bucket(bucket)
+ .key(key)
+ .build()));
+
+ }
+ }
+ }
+ }
+
+ static void abortMultipartUpload(String bucket, Date date, S3Client
s3Client) {
+ ListMultipartUploadsIterable multiUploadsResponse =
s3Client.listMultipartUploadsPaginator(listReq ->
+ listReq.bucket(bucket).build());
+ for (MultipartUpload multipartUpload : multiUploadsResponse.uploads())
{
+ Instant initiated = multipartUpload.initiated();
+ if (initiated.isBefore(date.toInstant())) {
+ s3Client.abortMultipartUpload(abortReq -> abortReq
+ .bucket(bucket)
+ .key(multipartUpload.key())
+ .uploadId(multipartUpload.uploadId())
+ .build());
+ }
+ }
+ }
+
+ /**
+ * Checks if the specified Amazon S3 bucket exists and is accessible with
the provided S3 client.
+ * <p>
+ * This method attempts to perform a {@code headBucket} request. If the
bucket exists and is accessible,
+ * it returns {@code true}. If the bucket does not exist, it returns
{@code false}.
+ * Other unexpected exceptions (such as permission errors or network
issues) will propagate.
+ * </p>
+ *
+ * @param s3Client the {@link S3Client} to use for the request
+ * @param bucketName the name of the S3 bucket to check
+ * @return {@code true} if the bucket exists and is accessible; {@code
false} if the bucket does not exist
+ * @throws NullPointerException if {@code s3Client} or {@code bucketName}
is null
+ * @throws S3Exception if an AWS error other than {@link
NoSuchBucketException} occurs
+ */
+ static boolean bucketExists(final S3Client s3Client, final String
bucketName) {
+ try {
+ s3Client.headBucket(request -> request.bucket(bucketName));
+ return true;
+ }
+ catch (NoSuchBucketException exception) {
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a specific object exists in the given S3 bucket.
+ * <p>
+ * Performs a {@code headObject} request using the provided {@link
S3RequestDecorator}.
+ * Returns {@code true} if the object exists, {@code false} if it does not
(404 or 403).
+ * Other {@link S3Exception}s are propagated.
+ * </p>
+ *
+ * @param s3Client the {@link S3Client} to use for the request
+ * @param bucket the S3 bucket name
+ * @param key the object key
+ * @param s3ReqDecorator decorator for the {@link HeadObjectRequest}
+ * @return {@code true} if the object exists, {@code false} if not found
or forbidden
+ * @throws S3Exception for AWS errors other than 404 or 403
+ */
+ static boolean objectExists(final S3Client s3Client, final String bucket,
final String key, S3RequestDecorator s3ReqDecorator) {
+ try {
+
s3Client.headObject(s3ReqDecorator.decorate(HeadObjectRequest.builder().bucket(bucket).key(key).build()));
+
+ LOG.debug("Object {} exists in bucket {}", key, bucket);
+ return true;
+ } catch (S3Exception e) {
+ if (e.statusCode() == 404 || e.statusCode() == 403) {
+ LOG.info("Object {} doesn't exists in bucket {}", key, bucket);
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Constructs the S3 endpoint URI based on the provided properties,
acceleration flag, and region.
+ * <p>
+ * The method determines the endpoint as follows:
+ * <ol>
+ * <li>If the S3 endpoint property is set, uses it directly.</li>
+ * <li>Otherwise, constructs the endpoint using the region and
acceleration flag.</li>
+ * <li>The protocol is taken from the properties or defaults to
"https".</li>
+ * </ol>
+ *
+ * @param prop the properties containing S3 configuration
+ * @param accReq whether to use S3 acceleration endpoint
+ * @param region the AWS region
+ * @return the constructed S3 endpoint URI
+ */
+ @NotNull
+ static URI getEndPointUri(Properties prop, boolean accReq, String region) {
Review Comment:
This is not entirely based on properties value, we also need to check
whether acceleration is enabled on given bucket.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
