>From Murtadha Al Hubail <[email protected]>: Murtadha Al Hubail has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17613 )
Change subject: [ASTERIXDB-3196][*DB] Skip logs REDO on cloud deployments ...................................................................... [ASTERIXDB-3196][*DB] Skip logs REDO on cloud deployments - user model changes: no - storage format changes: no - interface changes: no Details: - Since no updates logs are generated on cloud deployments, we should skip logs REDO during partition promotion. - Ensure cloud prefixes do not start with file separator. Change-Id: If05ba888ea8327508d4cde00380a88d1f3494780 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17613 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Tested-by: Jenkins <[email protected]> --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java 3 files changed, 35 insertions(+), 5 deletions(-) Approvals: Ali Alsuliman: Looks good to me, approved Jenkins: Verified; Verified Objections: Anon. E. Moose #1000171: Violations found diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java index 1372016..0684442 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java @@ -131,8 +131,10 @@ (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository(); localResourceRepository.cleanup(partition); localResourceRepository.clearResourcesCache(); - final IRecoveryManager recoveryManager = appCtx.getTransactionSubsystem().getRecoveryManager(); - recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()), true); + if (!appCtx.isCloudDeployment()) { + final IRecoveryManager recoveryManager = appCtx.getTransactionSubsystem().getRecoveryManager(); + recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()), true); + } partitions.put(partition, new Object()); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java index b508b6c..4939ab5 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java @@ -37,6 +37,7 @@ import org.apache.asterix.cloud.clients.CloudClientProvider; import org.apache.asterix.cloud.clients.ICloudClient; +import org.apache.asterix.cloud.clients.aws.s3.S3Utils; import org.apache.asterix.common.config.CloudProperties; import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -165,8 +166,7 @@ if (file.getFile().isDirectory()) { continue; } - - String path = file.getRelativePath(); + String path = S3Utils.toCloudPrefix(file.getRelativePath()); if (!cloudFiles.contains(path)) { // Delete local files that do not exist in cloud storage (the ground truth for valid files) localFilesIter.remove(); diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java index e3e9e98..4ea11b2 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.cloud.clients.aws.s3; +import java.io.File; import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; @@ -39,7 +40,7 @@ String newMarker = null; ListObjectsV2Response listObjectsResponse; ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(bucket); - listObjectsBuilder.prefix(encodeURI(path)); + listObjectsBuilder.prefix(encodeURI(toCloudPrefix(path))); while (true) { // List the objects from the start, or from the last marker in case of truncated result if (newMarker == null) { @@ -72,4 +73,8 @@ public static String decodeURI(String path) { return URLDecoder.decode(path, Charset.defaultCharset()); } + + public static String toCloudPrefix(String path) { + return path.startsWith(File.separator) ? path.substring(1) : path; + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17613 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: If05ba888ea8327508d4cde00380a88d1f3494780 Gerrit-Change-Number: 17613 Gerrit-PatchSet: 2 Gerrit-Owner: Murtadha Al Hubail <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Al Hubail <[email protected]> Gerrit-MessageType: merged
