This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9ee8d95fc8c [enhancement](cloud) make file cache background task
interval configurable (#49832)
9ee8d95fc8c is described below
commit 9ee8d95fc8ce06412e202b60488390a7a3ecfbd1
Author: zhengyu <zhangzhen...@selectdb.com>
AuthorDate: Tue Apr 15 22:26:51 2025 +0800
[enhancement](cloud) make file cache background task interval configurable
(#49832)
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/io/cache/block_file_cache.cpp | 14 +++++++-------
be/test/io/cache/block_file_cache_test.cpp | 14 +++++++-------
4 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 989f677f808..61e14ee95d1 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1090,6 +1090,8 @@ DEFINE_mInt64(cache_lock_held_long_tail_threshold_us,
"30000000");
DEFINE_mBool(enable_file_cache_keep_base_compaction_output, "false");
DEFINE_mInt64(file_cache_remove_block_qps_limit, "1000");
DEFINE_mInt64(file_cache_background_gc_interval_ms, "100");
+DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000");
+DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "20000");
DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800");
DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 6d75679b9d9..6954c04f7d1 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1135,6 +1135,8 @@ DECLARE_mInt64(cache_lock_held_long_tail_threshold_us);
DECLARE_mBool(enable_file_cache_keep_base_compaction_output);
DECLARE_mInt64(file_cache_remove_block_qps_limit);
DECLARE_mInt64(file_cache_background_gc_interval_ms);
+DECLARE_mInt64(file_cache_background_monitor_interval_ms);
+DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms);
// inverted index searcher cache
// cache entry stay time after lookup
DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 52e36e42f9f..513ca838ec3 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -1773,9 +1773,9 @@ void BlockFileCache::check_need_evict_cache_in_advance() {
}
void BlockFileCache::run_background_monitor() {
- int64_t interval_time_seconds = 20;
while (!_close) {
- TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time",
&interval_time_seconds);
+ int64_t interval_ms =
config::file_cache_background_monitor_interval_ms;
+ TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time",
&interval_ms);
check_disk_resource_limit();
if (config::enable_evict_file_cache_in_advance) {
check_need_evict_cache_in_advance();
@@ -1785,7 +1785,7 @@ void BlockFileCache::run_background_monitor() {
{
std::unique_lock close_lock(_close_mtx);
- _close_cv.wait_for(close_lock,
std::chrono::seconds(interval_time_seconds));
+ _close_cv.wait_for(close_lock,
std::chrono::milliseconds(interval_ms));
if (_close) {
break;
}
@@ -1830,12 +1830,12 @@ void BlockFileCache::run_background_monitor() {
}
void BlockFileCache::run_background_ttl_gc() { // TODO(zhengyu): fix!
- int64_t interval_time_seconds = 20;
while (!_close) {
- TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time",
&interval_time_seconds);
+ int64_t interval_ms = config::file_cache_background_ttl_gc_interval_ms;
+ TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time",
&interval_ms);
{
std::unique_lock close_lock(_close_mtx);
- _close_cv.wait_for(close_lock,
std::chrono::seconds(interval_time_seconds));
+ _close_cv.wait_for(close_lock,
std::chrono::milliseconds(interval_ms));
if (_close) {
break;
}
@@ -1858,7 +1858,7 @@ void BlockFileCache::run_background_gc() {
FileCacheKey key;
size_t batch_count = 0;
while (!_close) {
- size_t interval_ms = config::file_cache_background_gc_interval_ms;
+ int64_t interval_ms = config::file_cache_background_gc_interval_ms;
size_t batch_limit = config::file_cache_remove_block_qps_limit *
interval_ms / 1000;
{
std::unique_lock close_lock(_close_mtx);
diff --git a/be/test/io/cache/block_file_cache_test.cpp
b/be/test/io/cache/block_file_cache_test.cpp
index 4f377c43bbd..94292664926 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -2173,7 +2173,7 @@ TEST_F(BlockFileCacheTest, ttl_normal) {
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
"BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1; },
&guard1);
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1000; },
&guard1);
sp->enable_processing();
TUniqueId query_id;
query_id.hi = 1;
@@ -2272,7 +2272,7 @@ TEST_F(BlockFileCacheTest, ttl_modify) {
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
"BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1; },
&guard1);
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1000; },
&guard1);
sp->enable_processing();
TUniqueId query_id;
query_id.hi = 1;
@@ -2355,7 +2355,7 @@ TEST_F(BlockFileCacheTest, ttl_modify_memory_storage) {
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
"BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1; },
&guard1);
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1000; },
&guard1);
sp->enable_processing();
TUniqueId query_id;
query_id.hi = 1;
@@ -2996,7 +2996,7 @@ TEST_F(BlockFileCacheTest, recyle_cache_async_ttl) {
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
"BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1; },
&guard1);
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1000; },
&guard1);
SyncPoint::CallbackGuard guard2;
sp->set_call_back(
"BlockFileCache::set_remove_batch",
@@ -3126,7 +3126,7 @@ TEST_F(BlockFileCacheTest, test_factory_1) {
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
"BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1; },
&guard1);
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1000; },
&guard1);
SyncPoint::CallbackGuard guard2;
sp->set_call_back(
"BlockFileCache::set_remove_batch",
@@ -4899,7 +4899,7 @@ TEST_F(BlockFileCacheTest, reset_capacity) {
sp->clear_call_back("BlockFileCache::set_sleep_time");
}};
sp->set_call_back("BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1;
});
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) =
1000; });
sp->set_call_back("BlockFileCache::set_remove_batch",
[](auto&& args) { *try_any_cast<int*>(args[0]) = 2; });
sp->enable_processing();
@@ -6751,7 +6751,7 @@ TEST_F(BlockFileCacheTest, evict_in_advance) {
SyncPoint::CallbackGuard guard1;
sp->set_call_back(
"BlockFileCache::set_sleep_time",
- [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1; },
&guard1);
+ [](auto&& args) { *try_any_cast<int64_t*>(args[0]) = 1000; },
&guard1);
sp->enable_processing();
fs::create_directories(cache_base_path);
TUniqueId query_id;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
