This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new cf0a663b43 [fix](cache file) fix auto run cache file cleaner when be
start (#22724)
cf0a663b43 is described below
commit cf0a663b4341e0866153ac734723e26ca1a2b0bc
Author: yujun <[email protected]>
AuthorDate: Thu Aug 10 10:21:09 2023 +0800
[fix](cache file) fix auto run cache file cleaner when be start (#22724)
fix auto run cache file cleaner when be start
---
be/src/common/config.h | 2 ++
be/src/olap/olap_server.cpp | 16 +++++++++++++---
be/src/olap/storage_engine.cpp | 8 ++++++++
3 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 3aab83398f..34246da4cc 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -232,6 +232,8 @@ CONF_mInt32(tablet_rowset_stale_sweep_time_sec, "300");
// garbage sweep policy
CONF_Int32(max_garbage_sweep_interval, "3600");
CONF_Int32(min_garbage_sweep_interval, "180");
+// garbage sweep every batch will sleep 1ms
+CONF_mInt32(garbage_sweep_batch_size, "100");
CONF_mInt32(snapshot_expire_time_sec, "172800");
// It is only a recommended value. When the disk space is insufficient,
// the file storage period under trash dose not have to comply with this
parameter.
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 28fd8e7629..848539d513 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -765,11 +765,21 @@ void StorageEngine::_cooldown_tasks_producer_callback() {
}
void StorageEngine::_cache_file_cleaner_tasks_producer_callback() {
- int64_t interval = config::generate_cache_cleaner_task_interval_sec;
- do {
+ while (true) {
+ int64_t interval = config::generate_cache_cleaner_task_interval_sec;
+ if (interval <= 0) {
+ interval = 10;
+ }
+ bool stop =
_stop_background_threads_latch.wait_for(std::chrono::seconds(interval));
+ if (stop) {
+ break;
+ }
+ if (config::generate_cache_cleaner_task_interval_sec <= 0) {
+ continue;
+ }
LOG(INFO) << "Begin to Clean cache files";
FileCacheManager::instance()->gc_file_caches();
- } while
(!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval)));
+ }
}
} // namespace doris
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 203ddafe0c..03182ecd5c 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -787,6 +787,7 @@ Status StorageEngine::_do_sweep(const std::string&
scan_root, const time_t& loca
return res;
}
+ int curr_sweep_batch_size = 0;
try {
// Sort pathes by name, that is by delete time.
std::vector<path> sorted_pathes;
@@ -822,6 +823,13 @@ Status StorageEngine::_do_sweep(const std::string&
scan_root, const time_t& loca
res = Status::Error<OS_ERROR>();
continue;
}
+
+ curr_sweep_batch_size++;
+ if (config::garbage_sweep_batch_size > 0 &&
+ curr_sweep_batch_size >= config::garbage_sweep_batch_size)
{
+ curr_sweep_batch_size = 0;
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
} else {
// Because files are ordered by filename, i.e. by create time,
so all the left files are not expired.
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
