This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 40e2d4811b additional retry options for google storage #5669 #5670
(#5686)
40e2d4811b is described below
commit 40e2d4811b31545906ffc8c9b6d95a9cb59ce100
Author: Bart Maertens <[email protected]>
AuthorDate: Mon Sep 8 15:26:30 2025 +0200
additional retry options for google storage #5669 #5670 (#5686)
* Add retry strategy to GCS client, fixes #5669
* additional retry options. fixes #5669
---------
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../apache/hop/vfs/gs/GoogleStorageFileSystem.java | 23 +++
.../hop/vfs/gs/config/GoogleCloudConfig.java | 33 +++-
.../hop/vfs/gs/config/GoogleCloudConfigPlugin.java | 220 +++++++++++++++++++++
.../gs/config/messages/messages_en_US.properties | 20 ++
4 files changed, 295 insertions(+), 1 deletion(-)
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
index 19528c0a8a..74a713b781 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
@@ -18,6 +18,7 @@
package org.apache.hop.vfs.gs;
+import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import java.util.Collection;
@@ -28,6 +29,9 @@ import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.provider.AbstractFileName;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.hop.vfs.gs.config.GoogleCloudConfig;
+import org.apache.hop.vfs.gs.config.GoogleCloudConfigSingleton;
+import org.threeten.bp.Duration;
public class GoogleStorageFileSystem extends AbstractFileSystem {
@@ -59,9 +63,28 @@ public class GoogleStorageFileSystem extends
AbstractFileSystem {
return storage;
}
+ GoogleCloudConfig config = GoogleCloudConfigSingleton.getConfig();
+
+ RetrySettings retrySettings =
+ StorageOptions.getDefaultRetrySettings().toBuilder()
+ .setMaxAttempts(Integer.parseInt(config.getMaxAttempts()))
+ .setInitialRetryDelay(
+
Duration.ofSeconds(Integer.parseInt(config.getInitialRetryDelay())))
+
.setRetryDelayMultiplier(Double.parseDouble(config.getRetryDelayMultiplier()))
+
.setMaxRetryDelay(Duration.ofSeconds(Integer.parseInt(config.getMaxRetryDelay())))
+
.setTotalTimeout(Duration.ofMinutes(Integer.parseInt(config.getTotalTimeout())))
+ .setInitialRpcTimeout(
+
Duration.ofSeconds(Integer.parseInt(config.getInitialRpcTimeout())))
+
.setRpcTimeoutMultiplier(Double.parseDouble(config.getRpcTimeoutMultiplier()))
+ // max RPC Timeout setting causes problems, disabled for now
+ //
.setMaxRpcTimeout(Duration.ofSeconds(Integer.parseInt(config.getMaxRpcTimeout())))
+ .build();
+
StorageOptions.Builder optionsBuilder = StorageOptions.newBuilder();
optionsBuilder.setCredentials(
GoogleStorageFileSystemConfigBuilder.getInstance().getGoogleCredentials(fileSystemOptions));
+ optionsBuilder.setRetrySettings(retrySettings);
+
return storage = optionsBuilder.build().getService();
}
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfig.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfig.java
index 98b77be0c6..c4a0023b3d 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfig.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfig.java
@@ -27,11 +27,42 @@ public class GoogleCloudConfig {
public static final String HOP_CONFIG_GOOGLE_CLOUD_CONFIG_KEY =
"googleCloud";
private String serviceAccountKeyFile;
+ private String maxAttempts;
+ private String initialRetryDelay;
+ private String retryDelayMultiplier;
+ private String maxRetryDelay;
+ private String totalTimeout;
+ private String initialRpcTimeout;
+ private String rpcTimeoutMultiplier;
+ private String maxRpcTimeout;
+ private String connectionTimeout;
+ private String readTimeout;
- public GoogleCloudConfig() {}
+ public GoogleCloudConfig() {
+ maxAttempts = "6";
+ initialRetryDelay = "1";
+ retryDelayMultiplier = "2.0";
+ maxRetryDelay = "32";
+ totalTimeout = "50";
+ initialRpcTimeout = "50";
+ rpcTimeoutMultiplier = "1.0";
+ maxRpcTimeout = "50";
+ connectionTimeout = "20";
+ readTimeout = "20";
+ }
public GoogleCloudConfig(GoogleCloudConfig config) {
this();
serviceAccountKeyFile = config.serviceAccountKeyFile;
+ maxAttempts = config.maxAttempts;
+ initialRetryDelay = config.initialRetryDelay;
+ retryDelayMultiplier = config.retryDelayMultiplier;
+ maxRetryDelay = config.maxRetryDelay;
+ totalTimeout = config.totalTimeout;
+ initialRpcTimeout = config.initialRpcTimeout;
+ rpcTimeoutMultiplier = config.rpcTimeoutMultiplier;
+ maxRpcTimeout = config.maxRpcTimeout;
+ connectionTimeout = config.connectionTimeout;
+ readTimeout = config.readTimeout;
}
}
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfigPlugin.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfigPlugin.java
index cbe425622e..5780006a3a 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfigPlugin.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/config/GoogleCloudConfigPlugin.java
@@ -51,6 +51,26 @@ public class GoogleCloudConfigPlugin implements
IConfigOptions, IGuiPluginCompos
private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_ACCOUNT_KEY_FILE =
"10000-google-cloud-service-account-key-file";
+ private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_ATTEMPTS =
+ "10100-google-cloud-service-max-attempts";
+ private static final String
WIDGET_ID_GOOGLE_CLOUD_SERVICE_INITIAL_RETRY_DELAY =
+ "10200-google-cloud-service-initial-retry-delay";
+ private static final String
WIDGET_ID_GOOGLE_CLOUD_SERVICE_RETRY_DELAY_MULTIPLIER =
+ "10300-google-cloud-service-retry-delay-multiplier";
+ private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_RETRY_DELAY =
+ "10400-google-cloud-service-max-retry-delay";
+ private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_TOTAL_TIMEOUT =
+ "10500-google-cloud-service-total-timeout";
+ private static final String
WIDGET_ID_GOOGLE_CLOUD_SERVICE_INITIAL_RPC_TIMEOUT =
+ "10600-google-cloud-service-inital-rpc-timeout";
+ private static final String
WIDGET_ID_GOOGLE_CLOUD_SERVICE_RPC_TIMEOUT_MULTIPLIER =
+ "10700-google-cloud-service-rpc-timeout-multiplier";
+ private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_RPC_TIMEOUT =
+ "10800-google-cloud-service-max-rpc-timeout";
+ private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_CONNECT_TIMEOUT =
+ "10900-google-cloud-service-connect-timeout";
+ private static final String WIDGET_ID_GOOGLE_CLOUD_SERVICE_READ_TIMEOUT =
+ "1100-google-cloud-service-read-timeout";
@GuiWidgetElement(
id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_ACCOUNT_KEY_FILE,
@@ -64,6 +84,96 @@ public class GoogleCloudConfigPlugin implements
IConfigOptions, IGuiPluginCompos
description = "Configure the path to a Google Cloud service account JSON
key file")
private String serviceAccountKeyFile;
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_ATTEMPTS,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.MaxAttempts.Label",
+ toolTip = "i18n::GoogleCloudPlugin.MaxAttempts.Description")
+ private String maxAttempts;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_INITIAL_RETRY_DELAY,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.InitialRetryDelay.Label",
+ toolTip = "i18n::GoogleCloudPlugin.InitialRetryDelay.Description")
+ private String initialRetryDelay;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_RETRY_DELAY_MULTIPLIER,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.RetryDelayMultiplier.Label",
+ toolTip = "i18n::GoogleCloudPlugin.RetryDelayMultiplier.Description")
+ private String retryDelayMultiplier;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_RETRY_DELAY,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.MaxRetryDelay.Label",
+ toolTip = "i18n::GoogleCloudPlugin.MaxRetryDelay.Description")
+ private String maxRetryDelay;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_TOTAL_TIMEOUT,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.TotalTimeout.Label",
+ toolTip = "i18n::GoogleCloudPlugin.TotalTimeout.Description")
+ private String totalTimeout;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_INITIAL_RPC_TIMEOUT,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.InitialRpcTimeout.Label",
+ toolTip = "i18n::GoogleCloudPlugin.InitialRpcTimeout.Description")
+ private String initialRpcTimeout;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_RPC_TIMEOUT_MULTIPLIER,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.RpcTimeoutMultiplier.Label",
+ toolTip = "i18n::GoogleCloudPlugin.RpcTimeoutMultiplier.Description")
+ private String rpcTimeoutMultiplier;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_RPC_TIMEOUT,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.MaxRpcTimeout.Label",
+ toolTip = "i18n::GoogleCloudPlugin.MaxRpcTimeout.Description")
+ private String maxRpcTimeout;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_CONNECT_TIMEOUT,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.ConnectTimeout.Label",
+ toolTip = "i18n::GoogleCloudPlugin.ConnectTimeout.Description")
+ private String connectTimeout;
+
+ @GuiWidgetElement(
+ id = WIDGET_ID_GOOGLE_CLOUD_SERVICE_READ_TIMEOUT,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.TEXT,
+ variables = true,
+ label = "i18n::GoogleCloudPlugin.ReadTimeout.Label",
+ toolTip = "i18n::GoogleCloudPlugin.ReadTimeout.Description")
+ private String readTimeout;
+
/**
* Gets instance
*
@@ -74,6 +184,16 @@ public class GoogleCloudConfigPlugin implements
IConfigOptions, IGuiPluginCompos
GoogleCloudConfig config = GoogleCloudConfigSingleton.getConfig();
instance.serviceAccountKeyFile = config.getServiceAccountKeyFile();
+ instance.maxAttempts = config.getMaxAttempts();
+ instance.initialRetryDelay = config.getInitialRetryDelay();
+ instance.retryDelayMultiplier = config.getRetryDelayMultiplier();
+ instance.maxRetryDelay = config.getMaxRetryDelay();
+ instance.totalTimeout = config.getTotalTimeout();
+ instance.initialRpcTimeout = config.getInitialRpcTimeout();
+ instance.rpcTimeoutMultiplier = config.getRpcTimeoutMultiplier();
+ instance.maxRpcTimeout = config.getMaxRpcTimeout();
+ instance.connectTimeout = config.getConnectionTimeout();
+ instance.readTimeout = config.getReadTimeout();
return instance;
}
@@ -92,6 +212,67 @@ public class GoogleCloudConfigPlugin implements
IConfigOptions, IGuiPluginCompos
"The Google Cloud service account JSON jey file is set to '"
+ serviceAccountKeyFile
+ "'");
+
+ changed = true;
+ }
+
+ if (maxAttempts != null) {
+ config.setMaxAttempts(maxAttempts);
+ log.logBasic("Google Cloud service max attempts set to " +
maxAttempts);
+ changed = true;
+ }
+
+ if (initialRetryDelay != null) {
+ config.setInitialRetryDelay(initialRetryDelay);
+ log.logBasic("Google Cloud service initialRetryDelay set to " +
initialRetryDelay);
+ changed = true;
+ }
+
+ if (retryDelayMultiplier != null) {
+ config.setRetryDelayMultiplier(retryDelayMultiplier);
+ log.logBasic("Google Cloud service retryDelayMultiplier set to " +
retryDelayMultiplier);
+ changed = true;
+ }
+
+ if (maxRetryDelay != null) {
+ config.setMaxRetryDelay(maxRetryDelay);
+ log.logBasic("Google Cloud service maxRetryDelay set to " +
maxRetryDelay);
+ changed = true;
+ }
+
+ if (totalTimeout != null) {
+ config.setTotalTimeout(totalTimeout);
+ log.logBasic("Google Cloud service totalTimeout set to " +
totalTimeout);
+ changed = true;
+ }
+
+ if (initialRpcTimeout != null) {
+ config.setInitialRpcTimeout(initialRpcTimeout);
+ log.logBasic("Google Cloud service initialRpcTimeout set to " +
initialRpcTimeout);
+ changed = true;
+ }
+
+ if (rpcTimeoutMultiplier != null) {
+ config.setRpcTimeoutMultiplier(rpcTimeoutMultiplier);
+ log.logBasic("Google Cloud service rpcTimeoutMultiplier set to " +
rpcTimeoutMultiplier);
+ changed = true;
+ }
+
+ if (maxRpcTimeout != null) {
+ config.setMaxRpcTimeout(maxRpcTimeout);
+ log.logBasic("Google Cloud service maxRpcTimeout set to " +
maxRpcTimeout);
+ changed = true;
+ }
+
+ if (connectTimeout != null) {
+ config.setConnectionTimeout(connectTimeout);
+ log.logBasic("Google Cloud service connectTimeout set to " +
connectTimeout);
+ changed = true;
+ }
+
+ if (readTimeout != null) {
+ config.setReadTimeout(readTimeout);
+ log.logBasic("Google Cloud service readTimeout set to " + readTimeout);
changed = true;
}
@@ -131,6 +312,45 @@ public class GoogleCloudConfigPlugin implements
IConfigOptions, IGuiPluginCompos
serviceAccountKeyFile = ((TextVar) control).getText();
GoogleCloudConfigSingleton.getConfig().setServiceAccountKeyFile(serviceAccountKeyFile);
break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_ATTEMPTS:
+ maxAttempts = ((TextVar) control).getText();
+ GoogleCloudConfigSingleton.getConfig().setMaxAttempts(maxAttempts);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_INITIAL_RETRY_DELAY:
+ initialRetryDelay = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setInitialRetryDelay(initialRetryDelay);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_RETRY_DELAY_MULTIPLIER:
+ retryDelayMultiplier = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setRetryDelayMultiplier(retryDelayMultiplier);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_RETRY_DELAY:
+ maxRetryDelay = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setMaxRetryDelay(maxRetryDelay);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_TOTAL_TIMEOUT:
+ totalTimeout = ((TextVar) control).getText();
+ GoogleCloudConfigSingleton.getConfig().setTotalTimeout(totalTimeout);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_INITIAL_RPC_TIMEOUT:
+ initialRpcTimeout = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setInitialRpcTimeout(initialRpcTimeout);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_RPC_TIMEOUT_MULTIPLIER:
+ rpcTimeoutMultiplier = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setRpcTimeoutMultiplier(rpcTimeoutMultiplier);
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_MAX_RPC_TIMEOUT:
+ maxRpcTimeout = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setMaxRpcTimeout(maxRpcTimeout);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_CONNECT_TIMEOUT:
+ connectTimeout = ((TextVar) control).getText();
+
GoogleCloudConfigSingleton.getConfig().setConnectionTimeout(connectTimeout);
+ break;
+ case WIDGET_ID_GOOGLE_CLOUD_SERVICE_READ_TIMEOUT:
+ readTimeout = ((TextVar) control).getText();
+ GoogleCloudConfigSingleton.getConfig().setReadTimeout(readTimeout);
+ break;
}
}
// Save the project...
diff --git
a/plugins/tech/google/src/main/resources/org/apache/hop/vfs/gs/config/messages/messages_en_US.properties
b/plugins/tech/google/src/main/resources/org/apache/hop/vfs/gs/config/messages/messages_en_US.properties
index 4469d8b7be..99813db674 100644
---
a/plugins/tech/google/src/main/resources/org/apache/hop/vfs/gs/config/messages/messages_en_US.properties
+++
b/plugins/tech/google/src/main/resources/org/apache/hop/vfs/gs/config/messages/messages_en_US.properties
@@ -20,3 +20,23 @@
GoogleCloudPlugin.AccountKeyFile.Description=The path to a Google Cloud
service account JSON key file
GoogleCloudPlugin.AccountKeyFile.Label=Account key file
GoogleCloudPlugin.GuiPlugin.Description=Google Cloud
+GoogleCloudPlugin.MaxAttempts.Label=Max number of attempts
+GoogleCloudPlugin.MaxAttempts.Description=Max number of attempts
+GoogleCloudPlugin.InitialRetryDelay.Label=Initial retry delay
+GoogleCloudPlugin.InitialRetryDelay.Description=Initial retry delay
+GoogleCloudPlugin.RetryDelayMultiplier.Label=Retry delay multiplier
+GoogleCloudPlugin.RetryDelayMultiplier.Description=Retry delay multiplier
+GoogleCloudPlugin.MaxRetryDelay.Label=Maximum retry delay
+GoogleCloudPlugin.MaxRetryDelay.Description=Maximum retry delay
+GoogleCloudPlugin.TotalTimeout.Label=Total Timeout
+GoogleCloudPlugin.TotalTimeout.Description=Total Timeout
+GoogleCloudPlugin.InitialRpcTimeout.Label=Initial RPC Timeout
+GoogleCloudPlugin.InitialRpcTimeout.Description=Initial RPC Timeout
+GoogleCloudPlugin.RpcTimeoutMultiplier.Label=RPC Timeout Multiplier
+GoogleCloudPlugin.RpcTimeoutMultiplier.Description=RPC Timeout Multiplier
+GoogleCloudPlugin.MaxRpcTimeout.Label=Max RPC Timeout
+GoogleCloudPlugin.MaxRpcTimeout.Description=Max RPC Timeout
+GoogleCloudPlugin.ConnectTimeout.Label=Connect Timeout
+GoogleCloudPlugin.ConnectTimeout.Description=Connect Timeout
+GoogleCloudPlugin.ReadTimeout.Label=Read Timeout
+GoogleCloudPlugin.ReadTimeout.Description=Read Timeout
\ No newline at end of file
