This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94d563f04d [improvement](garbage sweep) garbage sweep sleep for a
while to reduce io (#22762)
94d563f04d is described below
commit 94d563f04d37f618132a73ad3872c271352ecbe7
Author: yujun <[email protected]>
AuthorDate: Thu Aug 10 12:11:50 2023 +0800
[improvement](garbage sweep) garbage sweep sleep for a while to reduce io
(#22762)
---
be/src/common/config.cpp | 1 +
be/src/common/config.h | 2 ++
be/src/olap/olap_server.cpp | 16 +++++++++++++---
be/src/olap/storage_engine.cpp | 8 ++++++++
4 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ad7ed6614c..4150cce384 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -289,6 +289,7 @@ DEFINE_mInt32(tablet_rowset_stale_sweep_time_sec, "300");
// garbage sweep policy
DEFINE_Int32(max_garbage_sweep_interval, "3600");
DEFINE_Int32(min_garbage_sweep_interval, "180");
+DEFINE_mInt32(garbage_sweep_batch_size, "100");
DEFINE_mInt32(snapshot_expire_time_sec, "172800");
// It is only a recommended value. When the disk space is insufficient,
// the file storage period under trash dose not have to comply with this
parameter.
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 416eae2880..e835e759af 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -335,6 +335,8 @@ DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec);
// garbage sweep policy
DECLARE_Int32(max_garbage_sweep_interval);
DECLARE_Int32(min_garbage_sweep_interval);
+// garbage sweep every batch will sleep 1ms
+DECLARE_mInt32(garbage_sweep_batch_size);
DECLARE_mInt32(snapshot_expire_time_sec);
// It is only a recommended value. When the disk space is insufficient,
// the file storage period under trash dose not have to comply with this
parameter.
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 1b4f9a47cb..e698980a9c 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1196,11 +1196,21 @@ void
StorageEngine::_cold_data_compaction_producer_callback() {
}
void StorageEngine::_cache_file_cleaner_tasks_producer_callback() {
- int64_t interval = config::generate_cache_cleaner_task_interval_sec;
- do {
+ while (true) {
+ int64_t interval = config::generate_cache_cleaner_task_interval_sec;
+ if (interval <= 0) {
+ interval = 10;
+ }
+ bool stop =
_stop_background_threads_latch.wait_for(std::chrono::seconds(interval));
+ if (stop) {
+ break;
+ }
+ if (config::generate_cache_cleaner_task_interval_sec <= 0) {
+ continue;
+ }
LOG(INFO) << "Begin to Clean cache files";
FileCacheManager::instance()->gc_file_caches();
- } while
(!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval)));
+ }
}
void StorageEngine::add_async_publish_task(int64_t partition_id, int64_t
tablet_id,
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 98d1a259e8..63f2224c55 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -897,6 +897,7 @@ Status StorageEngine::_do_sweep(const std::string&
scan_root, const time_t& loca
return res;
}
+ int curr_sweep_batch_size = 0;
try {
// Sort pathes by name, that is by delete time.
std::vector<path> sorted_pathes;
@@ -928,6 +929,13 @@ Status StorageEngine::_do_sweep(const std::string&
scan_root, const time_t& loca
if (!res.ok()) {
continue;
}
+
+ curr_sweep_batch_size++;
+ if (config::garbage_sweep_batch_size > 0 &&
+ curr_sweep_batch_size >= config::garbage_sweep_batch_size)
{
+ curr_sweep_batch_size = 0;
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
} else {
// Because files are ordered by filename, i.e. by create time,
so all the left files are not expired.
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
