>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19525 )
Change subject: [ASTERIXDB-3581][EXT]: Do not retry GCS SDK requests if thread
is interrupted
......................................................................
[ASTERIXDB-3581][EXT]: Do not retry GCS SDK requests if thread is interrupted
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Do not retry any GCS requests if the thread
has been interrupted.
Ext-ref: MB-65548
Change-Id: I5deb956bda11ee2ca41a1c05616788b2d697622f
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSParallelDownloader.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
4 files changed, 84 insertions(+), 1 deletion(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/25/19525/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index 16fb278..1f93d05 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -19,6 +19,7 @@
package org.apache.asterix.cloud.clients.google.gcs;
import static
org.apache.asterix.cloud.clients.google.gcs.GCSClientConfig.DELETE_BATCH_SIZE;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -328,6 +329,7 @@
private static Storage buildClient(GCSClientConfig config) throws
HyracksDataException {
StorageOptions.Builder builder =
StorageOptions.newBuilder().setCredentials(config.createCredentialsProvider());
+
builder.setStorageRetryStrategy(DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY);
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.setHost(config.getEndpoint());
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSParallelDownloader.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSParallelDownloader.java
index cbbaf31..4426d61 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSParallelDownloader.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSParallelDownloader.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.cloud.clients.google.gcs;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY;
+
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -64,6 +66,7 @@
this.ioManager = ioManager;
this.profiler = profiler;
StorageOptions.Builder builder = StorageOptions.newBuilder();
+
builder.setStorageRetryStrategy(DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY);
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.setHost(config.getEndpoint());
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
index 2613f34..51648d5 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
@@ -18,7 +18,14 @@
*/
package org.apache.asterix.external.util.google.gcs;
-public class GCSConstants {
+import java.util.concurrent.CancellationException;
+
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.gax.retrying.TimedAttemptSettings;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.storage.StorageRetryStrategy;
+
+public class GCSConstants<ResponseT> {
private GCSConstants() {
throw new AssertionError("do not instantiate");
}
@@ -57,4 +64,55 @@
public static final String PRIVATE_KEY =
"fs.gs.auth.service.account.private.key";
public static final String CLIENT_EMAIL =
"fs.gs.auth.service.account.email";
}
+
+ public static final StorageRetryStrategy
DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY;
+ static {
+ StorageRetryStrategy defaultStrategy =
StorageRetryStrategy.getDefaultStorageRetryStrategy();
+ ExceptionHandler defaultIdempotentHandler = (ExceptionHandler)
defaultStrategy.getIdempotentHandler();
+ ExceptionHandler defaultNonIdempotentHandler = (ExceptionHandler)
defaultStrategy.getNonidempotentHandler();
+
+ ResultRetryAlgorithm<Object> noRetryOnThreadInterruptIdempotentHandler
= new ResultRetryAlgorithm<>() {
+ @Override
+ public TimedAttemptSettings createNextAttempt(Throwable
prevThrowable, Object prevResponse,
+ TimedAttemptSettings prevSettings) {
+ return
defaultIdempotentHandler.createNextAttempt(prevThrowable, prevResponse,
prevSettings);
+ }
+
+ @Override
+ public boolean shouldRetry(Throwable prevThrowable, Object
prevResponse) throws CancellationException {
+ if (Thread.currentThread().isInterrupted()) {
+ return false;
+ }
+ return defaultIdempotentHandler.shouldRetry(prevThrowable,
prevResponse);
+ }
+ };
+
+ ResultRetryAlgorithm<Object>
noRetryOnThreadInterruptNonIdempotentHandler = new ResultRetryAlgorithm<>() {
+ @Override
+ public TimedAttemptSettings createNextAttempt(Throwable
prevThrowable, Object prevResponse,
+ TimedAttemptSettings prevSettings) {
+ return
defaultNonIdempotentHandler.createNextAttempt(prevThrowable, prevResponse,
prevSettings);
+ }
+
+ @Override
+ public boolean shouldRetry(Throwable prevThrowable, Object
prevResponse) throws CancellationException {
+ if (Thread.currentThread().isInterrupted()) {
+ return false;
+ }
+ return defaultNonIdempotentHandler.shouldRetry(prevThrowable,
prevResponse);
+ }
+ };
+
+ DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY = new
StorageRetryStrategy() {
+ @Override
+ public ResultRetryAlgorithm<?> getIdempotentHandler() {
+ return noRetryOnThreadInterruptIdempotentHandler;
+ }
+
+ @Override
+ public ResultRetryAlgorithm<?> getNonidempotentHandler() {
+ return noRetryOnThreadInterruptNonIdempotentHandler;
+ }
+ };
+ }
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
index 481b7ff..c639d13 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -26,6 +26,7 @@
import static
org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
import static
org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.ENDPOINT_FIELD_NAME;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_TYPE;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_UNAUTHENTICATED;
@@ -94,6 +95,7 @@
String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
StorageOptions.Builder builder = StorageOptions.newBuilder();
+
builder.setStorageRetryStrategy(DEFAULT_NO_RETRY_ON_THREAD_INTERRUPT_STRATEGY);
// default credentials provider
if (applicationDefaultCredentials != null) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19525
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: ionic
Gerrit-Change-Id: I5deb956bda11ee2ca41a1c05616788b2d697622f
Gerrit-Change-Number: 19525
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
