This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new f402bb732d1 branch-4.1: [chore](s3) Adjust S3 SlowDown retry
configurable #65508 (#65641)
f402bb732d1 is described below
commit f402bb732d12743df5617a4a6e495331d2d74184
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Jul 16 11:02:52 2026 +0800
branch-4.1: [chore](s3) Adjust S3 SlowDown retry configurable #65508
(#65641)
Cherry-picked from #65508
Co-authored-by: Yixuan Wang <[email protected]>
---
be/src/util/s3_util.cpp | 3 ++-
cloud/src/common/config.h | 2 --
cloud/src/recycler/s3_accessor.cpp | 6 ++----
3 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index ed6a21f3953..92f7a006c10 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -349,6 +349,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_azure_client(
}
Azure::Storage::Blobs::BlobClientOptions options;
+
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
options.Retry.MaxRetries = config::max_s3_client_retry;
options.PerRetryPolicies.emplace_back(std::make_unique<AzureRetryRecordPolicy>());
if (_ca_cert_file_path.empty()) {
@@ -525,7 +526,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_s3_client(
}
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
- config::max_s3_client_retry /*scaleFactor = 25*/,
/*retry_slow_down=*/false);
+ config::max_s3_client_retry /*scaleFactor = 25*/,
/*retry_slow_down=*/true);
std::shared_ptr<Aws::S3::S3Client> new_client =
std::make_shared<Aws::S3::S3Client>(
get_aws_credentials_provider(s3_conf), std::move(aws_config),
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 31eed2188ff..c505f4a7b1b 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -314,8 +314,6 @@ CONF_Validator(s3_client_http_scheme, [](const std::string&
config) -> bool {
// Max retry times for object storage request
CONF_mInt64(max_s3_client_retry, "10");
-// Whether to retry on S3 SlowDown (429/503) errors
-CONF_Bool(s3_client_retry_slow_down, "false");
// Max byte getting delete bitmap can return, default is 1GB
CONF_mInt64(max_get_delete_bitmap_byte, "1073741824");
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 51cc9023904..8ff29694bd7 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -394,9 +394,6 @@ int S3Accessor::init() {
case S3Conf::AZURE: {
#ifdef USE_AZURE
Azure::Storage::Blobs::BlobClientOptions options;
- if (config::s3_client_retry_slow_down) {
-
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
- }
options.Retry.MaxRetries = config::max_s3_client_retry;
auto cred =
std::make_shared<Azure::Storage::StorageSharedKeyCredential>(conf_.ak,
conf_.sk);
@@ -441,8 +438,9 @@ int S3Accessor::init() {
if (config::s3_client_http_scheme == "http") {
aws_config.scheme = Aws::Http::Scheme::HTTP;
}
+ // Recycler should fail fast on S3 SlowDown instead of retrying and
blocking worker threads.
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
- config::max_s3_client_retry,
config::s3_client_retry_slow_down);
+ config::max_s3_client_retry, /*retry_slow_down=*/false);
if (_ca_cert_file_path.empty()) {
_ca_cert_file_path =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
