>From Ritik Raj <[email protected]>:
Ritik Raj has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19345 )
Change subject: wip: handle delete failure in batchOp
......................................................................
wip: handle delete failure in batchOp
Change-Id: Id59be58699ffbfd64cb4d1ebf496e166eae070e4
---
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
7 files changed, 70 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/45/19345/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index b208714..fd82944 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -124,7 +124,7 @@
* @param bucket bucket
* @param paths paths of all objects to be deleted
*/
- void deleteObjects(String bucket, Collection<String> paths);
+ void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException;
/**
* Returns the size of the object at the specified path
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
index 4e1c0f7..28fa53e 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
@@ -98,7 +98,7 @@
}
@Override
- public void deleteObjects(String bucket, Collection<String> paths) {
+ public void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException {
cloudClient.deleteObjects(bucket, paths);
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 319b713..4d8ffde 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -51,6 +51,8 @@
import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.util.annotations.ThreadSafe;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -65,16 +67,19 @@
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Error;
import software.amazon.awssdk.services.s3.model.S3Object;
@ThreadSafe
public final class S3CloudClient implements ICloudClient {
+ private static final Logger LOGGER = LogManager.getLogger();
private final S3ClientConfig config;
private final S3Client s3Client;
private final ICloudGuardian guardian;
@@ -216,7 +221,7 @@
}
@Override
- public void deleteObjects(String bucket, Collection<String> paths) {
+ public void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException {
if (paths.isEmpty()) {
return;
}
@@ -234,7 +239,15 @@
Delete delete =
Delete.builder().objects(objectIdentifiers).build();
DeleteObjectsRequest deleteReq =
DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
- s3Client.deleteObjects(deleteReq);
+ DeleteObjectsResponse deleteObjectsResponse =
s3Client.deleteObjects(deleteReq);
+ if (deleteObjectsResponse.hasErrors()) {
+ List<S3Error> deleteErrors = deleteObjectsResponse.errors();
+ for (S3Error s3Error : deleteErrors) {
+ LOGGER.warn("Failed to delete object: {}, code: {},
message: {}", s3Error.key(), s3Error.code(),
+ s3Error.message());
+ }
+ throw new HyracksDataException("Failed to delete objects");
+ }
profiler.objectDelete();
}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index b9f9421..3d7880b 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -48,6 +48,8 @@
import org.apache.asterix.cloud.clients.profiler.CountRequestProfilerLimiter;
import org.apache.asterix.cloud.clients.profiler.IRequestProfilerLimiter;
import org.apache.asterix.cloud.clients.profiler.RequestLimiterNoOpProfiler;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.control.nc.io.IOManager;
@@ -55,6 +57,7 @@
import org.apache.logging.log4j.Logger;
import com.azure.core.http.rest.PagedIterable;
+import com.azure.core.http.rest.Response;
import com.azure.core.util.BinaryData;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
@@ -79,6 +82,7 @@
private static final String AZURITE_ACCOUNT_NAME = "devstoreaccount1";
private static final String AZURITE_ACCOUNT_KEY =
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+ private static final int HTTP_STATUS_ACCEPTED = 202;
private final ICloudGuardian guardian;
private BlobContainerClient blobContainerClient;
private AzBlobStorageClientConfig config;
@@ -246,7 +250,7 @@
}
@Override
- public void deleteObjects(String bucket, Collection<String> paths) {
+ public void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException {
if (paths.isEmpty())
return;
Set<BlobItem> blobsToDelete = getBlobsMatchingThesePaths(paths);
@@ -255,7 +259,17 @@
return;
Collection<List<String>> batchedBlobURLs =
getBatchedBlobURLs(blobURLs);
for (List<String> batch : batchedBlobURLs) {
- blobBatchClient.deleteBlobs(batch, null).stream().count();
+ PagedIterable<Response<Void>> responses =
blobBatchClient.deleteBlobs(batch, null);
+ Iterator<String> deletePathIter = paths.iterator();
+ String deletedPath = null;
+ try {
+ for (Response<Void> response : responses) {
+ deletedPath = deletePathIter.next();
+ response.getStatusCode();
+ }
+ } catch (BlobStorageException e) {
+ throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE, e,
"DELETE", paths.toString(), deletedPath);
+ }
}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index 62ca4ec..188cfe3 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -41,17 +41,22 @@
import org.apache.asterix.cloud.clients.profiler.CountRequestProfilerLimiter;
import org.apache.asterix.cloud.clients.profiler.IRequestProfilerLimiter;
import org.apache.asterix.cloud.clients.profiler.RequestLimiterNoOpProfiler;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.control.nc.io.IOManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.api.gax.paging.Page;
+import com.google.cloud.BaseServiceException;
import com.google.cloud.ReadChannel;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
@@ -60,10 +65,12 @@
import com.google.cloud.storage.Storage.BlobListOption;
import com.google.cloud.storage.Storage.CopyRequest;
import com.google.cloud.storage.StorageBatch;
+import com.google.cloud.storage.StorageBatchResult;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.StorageOptions;
public class GCSCloudClient implements ICloudClient {
+ private static final Logger LOGGER = LogManager.getLogger();
private final Storage gcsClient;
private final GCSClientConfig config;
private final ICloudGuardian guardian;
@@ -193,11 +200,12 @@
}
@Override
- public void deleteObjects(String bucket, Collection<String> paths) {
+ public void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException {
if (paths.isEmpty()) {
return;
}
+ List<StorageBatchResult<Boolean>> deleteResponses = new ArrayList<>();
StorageBatch batchRequest;
Iterator<String> pathIter = paths.iterator();
while (pathIter.hasNext()) {
@@ -205,10 +213,25 @@
for (int i = 0; pathIter.hasNext() && i < DELETE_BATCH_SIZE; i++) {
BlobId blobId = BlobId.of(bucket, config.getPrefix() +
pathIter.next());
guardian.checkWriteAccess(bucket, blobId.getName());
- batchRequest.delete(blobId);
+ deleteResponses.add(batchRequest.delete(blobId));
}
batchRequest.submit();
+ Iterator<String> deletePathIter = paths.iterator();
+ for (StorageBatchResult<Boolean> deleteResponse : deleteResponses)
{
+ String deletedPath = deletePathIter.next();
+ try {
+ boolean deleted = deleteResponse.get();
+ if (!deleted) {
+ throw new
RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE, "DELETE", paths.toString(),
+ deletedPath);
+ }
+ } catch (BaseServiceException e) {
+ LOGGER.warn("Failed to delete object {} while deleting
{}", deletedPath, paths, e);
+ throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE,
e, "DELETE", paths.toString(),
+ deletedPath);
+ }
+ }
profilerLimiter.objectDelete();
}
}
@@ -287,4 +310,4 @@
private String stripCloudPrefix(String objectName) {
return objectName.substring(config.getPrefix().length());
}
-}
\ No newline at end of file
+}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index e131f8a..afc43e2 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -99,6 +99,7 @@
INVALID_KEY_TYPE(68),
FAILED_TO_READ_KEY(69),
AVRO_SUPPORTED_TYPE_WITH_OPTION(70),
+ CLOUD_IO_FAILURE(71),
UNSUPPORTED_JRE(100),
EXTERNAL_UDF_RESULT_TYPE_ERROR(200),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 16dcda5..99bc631 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -105,6 +105,7 @@
68 = Invalid key type. Expected '%1$s', found '%2$s'.
69 = Failed to read key. Reason: %1$s.
70 = Avro type '%1$s' is not supported by default. To enable type conversion,
recreate the external dataset with the option '%2$s' enabled
+71 = Cloud IO '%1$s' operation failed for files '%2$s' while deleting file
'%3$s'
100 = Unsupported JRE: %1$s
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19345
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Id59be58699ffbfd64cb4d1ebf496e166eae070e4
Gerrit-Change-Number: 19345
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <[email protected]>
Gerrit-MessageType: newchange
