This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 5c4a9c4e706 branch-4.1: [fix](cloud) make S3 rate limiter config take
effect dynamically in cloud mode #64554 (#64841)
5c4a9c4e706 is described below
commit 5c4a9c4e706baa19a3e3b5e81531c632dfeb5263
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 26 11:59:52 2026 +0800
branch-4.1: [fix](cloud) make S3 rate limiter config take effect
dynamically in cloud mode #64554 (#64841)
Cherry-picked from #64554
Co-authored-by: Xin Liao <[email protected]>
---
be/src/cloud/cloud_storage_engine.cpp | 9 +++++++
be/src/util/s3_util.h | 3 +++
be/test/util/s3_util_test.cpp | 45 +++++++++++++++++++++++++++++++++++
3 files changed, 57 insertions(+)
diff --git a/be/src/cloud/cloud_storage_engine.cpp
b/be/src/cloud/cloud_storage_engine.cpp
index 5383989346d..6ba92830f99 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -446,6 +446,15 @@ void
CloudStorageEngine::_refresh_storage_vault_info_thread_callback() {
while (!_stop_background_threads_latch.wait_for(
std::chrono::seconds(config::refresh_s3_info_interval_s))) {
sync_storage_vault();
+ // The other place that rebuilds the S3 rate limiter is
S3ClientFactory::create(), which
+ // is not called when an existing vault's conf is unchanged. Trigger
the check here as well
+ // so that dynamically modified s3_{get,put}_* rate limiter configs
take effect within
+ // refresh_s3_info_interval_s even when no vault is created or its
conf does not change.
+ // Gate it behind enable_s3_rate_limiter so that clusters with rate
limiting disabled
+ // (e.g. HDFS-only vaults) do not force-initialize S3ClientFactory /
the AWS SDK here.
+ if (config::enable_s3_rate_limiter) {
+ check_s3_rate_limiter_config_changed();
+ }
}
}
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 23c82c65e13..a721bcd28c1 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -64,6 +64,9 @@ extern bvar::LatencyRecorder s3_copy_object_latency;
std::string hide_access_key(const std::string& ak);
int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t
max_burst, size_t limit);
+// Rebuild the S3 GET/PUT rate limiters if the related configs have changed.
+// Safe to call periodically; it is a no-op when nothing changed.
+void check_s3_rate_limiter_config_changed();
class S3URI;
struct S3ClientConf {
diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp
index cd8eeb5cb68..a87d0e9a2d2 100644
--- a/be/test/util/s3_util_test.cpp
+++ b/be/test/util/s3_util_test.cpp
@@ -21,6 +21,7 @@
#include <string>
+#include "common/config.h"
#include "gtest/gtest_pred_impl.h"
#include "util/s3_uri.h"
@@ -76,4 +77,48 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) {
EXPECT_EQ("xxxxxxxFODNN7xxxxxxx", result);
}
+// Verifies that check_s3_rate_limiter_config_changed() rebuilds the global
GET rate
+// limiter when the related configs change. This is the behavior the cloud
vault refresh
+// thread relies on to apply dynamically modified s3_get_* rate limiter
configs without
+// having to (re)create an S3 client.
+TEST_F(S3UTILTest, check_s3_rate_limiter_config_changed_rebuilds_limiter) {
+ auto* get_limiter =
S3ClientFactory::instance().rate_limiter(S3RateLimitType::GET);
+ ASSERT_NE(get_limiter, nullptr);
+
+ // Save originals so other tests are not affected.
+ const int64_t orig_tps = config::s3_get_token_per_second;
+ const int64_t orig_bucket = config::s3_get_bucket_tokens;
+ const int64_t orig_limit = config::s3_get_token_limit;
+
+ // Establish a known baseline (no count limit, no throttling).
+ config::s3_get_token_per_second = 1000000000;
+ config::s3_get_bucket_tokens = 1000000000;
+ config::s3_get_token_limit = 0;
+ check_s3_rate_limiter_config_changed();
+
+ // Impose a hard request-count limit of 3. Since the limit value changes
(0 -> 3),
+ // the limiter is rebuilt with a fresh counter.
+ config::s3_get_token_limit = 3;
+ check_s3_rate_limiter_config_changed();
+
+ // The bucket/speed are huge so add() never throttles (returns 0); only
the count
+ // limit takes effect: the first 3 requests pass, the 4th is rejected (-1).
+ EXPECT_GE(get_limiter->add(1), 0);
+ EXPECT_GE(get_limiter->add(1), 0);
+ EXPECT_GE(get_limiter->add(1), 0);
+ EXPECT_LT(get_limiter->add(1), 0);
+
+ // Raise the limit. The checker must rebuild the limiter so the exhausted
counter is
+ // reset; otherwise the next request would still be rejected.
+ config::s3_get_token_limit = 100;
+ check_s3_rate_limiter_config_changed();
+ EXPECT_GE(get_limiter->add(1), 0);
+
+ // Restore original configs and apply them back to the limiter.
+ config::s3_get_token_per_second = orig_tps;
+ config::s3_get_bucket_tokens = orig_bucket;
+ config::s3_get_token_limit = orig_limit;
+ check_s3_rate_limiter_config_changed();
+}
+
} // end namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
