This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 205ae4a6ed5 [improve](load) reduce logs from memtable memory limiter
(#29840)
205ae4a6ed5 is described below
commit 205ae4a6ed511b90547681b885bfbd9ed618f081
Author: Kaijie Chen <[email protected]>
AuthorDate: Thu Jan 11 19:48:38 2024 +0800
[improve](load) reduce logs from memtable memory limiter (#29840)
---
be/src/olap/memtable_memory_limiter.cpp | 33 ++++++++++++++++++++++++---------
be/src/olap/memtable_memory_limiter.h | 7 ++++++-
2 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index 4b4298853a3..696ca4a953c 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -67,6 +67,7 @@ Status MemTableMemoryLimiter::init(int64_t process_mem_limit)
{
"MemTableMemoryLimiter");
REGISTER_HOOK_METRIC(memtable_memory_limiter_mem_consumption,
[this]() { return _mem_tracker->consumption(); });
+ _log_timer.start();
return Status::OK();
}
@@ -203,17 +204,31 @@ int64_t
MemTableMemoryLimiter::_flush_memtable(std::weak_ptr<MemTableWriter> wri
void MemTableMemoryLimiter::refresh_mem_tracker() {
std::lock_guard<std::mutex> l(_lock);
_refresh_mem_tracker();
+ std::stringstream ss;
+ Limit limit = Limit::NONE;
if (_soft_limit_reached()) {
- LOG(INFO) << "reached " << (_hard_limit_reached() ? "hard" : "soft")
<< " limit"
- << ", process mem: " << PerfCounters::get_vm_rss_str()
- << " (without allocator cache: "
- <<
PrettyPrinter::print_bytes(MemInfo::proc_mem_no_allocator_cache())
- << "), load mem: " <<
PrettyPrinter::print_bytes(_mem_tracker->consumption())
- << ", memtable writers num: " << _writers.size()
- << " (active: " <<
PrettyPrinter::print_bytes(_active_mem_usage)
- << ", write: " <<
PrettyPrinter::print_bytes(_write_mem_usage)
- << ", flush: " <<
PrettyPrinter::print_bytes(_flush_mem_usage) << ")";
+ limit = _hard_limit_reached() ? Limit::HARD : Limit::SOFT;
+ ss << "reached " << (limit == Limit::HARD ? "hard" : "soft") << "
limit";
+ } else if (_last_limit == Limit::NONE) {
+ return;
+ } else {
+ ss << "ended " << (_last_limit == Limit::HARD ? "hard" : "soft") << "
limit";
}
+
+ if (_last_limit == limit && _log_timer.elapsed_time() < LOG_INTERVAL) {
+ return;
+ }
+
+ _last_limit = limit;
+ _log_timer.reset();
+ LOG(INFO) << ss.str() << ", process mem: " <<
PerfCounters::get_vm_rss_str()
+ << " (without allocator cache: "
+ <<
PrettyPrinter::print_bytes(MemInfo::proc_mem_no_allocator_cache())
+ << "), load mem: " <<
PrettyPrinter::print_bytes(_mem_tracker->consumption())
+ << ", memtable writers num: " << _writers.size()
+ << " (active: " << PrettyPrinter::print_bytes(_active_mem_usage)
+ << ", write: " << PrettyPrinter::print_bytes(_write_mem_usage)
+ << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage)
<< ")";
}
void MemTableMemoryLimiter::_refresh_mem_tracker() {
diff --git a/be/src/olap/memtable_memory_limiter.h
b/be/src/olap/memtable_memory_limiter.h
index 7b91c9a9089..25a263675d7 100644
--- a/be/src/olap/memtable_memory_limiter.h
+++ b/be/src/olap/memtable_memory_limiter.h
@@ -22,6 +22,7 @@
#include "common/status.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "util/countdown_latch.h"
+#include "util/stopwatch.hpp"
namespace doris {
class MemTableWriter;
@@ -71,7 +72,11 @@ private:
int64_t _load_soft_mem_limit = -1;
int64_t _load_safe_mem_permit = -1;
+ enum Limit { NONE, SOFT, HARD } _last_limit;
+ MonotonicStopWatch _log_timer;
+ static const int64_t LOG_INTERVAL = 1 * 1000 * 1000 * 1000; // 1s
+
std::vector<std::weak_ptr<MemTableWriter>> _writers;
std::vector<std::weak_ptr<MemTableWriter>> _active_writers;
};
-} // namespace doris
\ No newline at end of file
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
