This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit aa156f0781fadd2844f9e9429a57d0ae3333e98b Author: Xinyi Zou <[email protected]> AuthorDate: Mon May 6 20:06:18 2024 +0800 [opt](memory) BE memory info compatible with Cgroup (#34262) --- be/src/util/cgroup_util.cpp | 11 ++++ be/src/util/cgroup_util.h | 4 ++ be/src/util/mem_info.cpp | 127 +++++++++++++++++++++++++------------------- be/src/util/mem_info.h | 35 +++++++----- 4 files changed, 108 insertions(+), 69 deletions(-) diff --git a/be/src/util/cgroup_util.cpp b/be/src/util/cgroup_util.cpp index b1cb9cf8ead..02337392253 100644 --- a/be/src/util/cgroup_util.cpp +++ b/be/src/util/cgroup_util.cpp @@ -167,6 +167,17 @@ Status CGroupUtil::find_cgroup_mem_limit(int64_t* bytes) { string limit_file_path = cgroup_path + "/memory.limit_in_bytes"; return read_cgroup_value(limit_file_path, bytes); } + +Status CGroupUtil::find_cgroup_mem_usage(int64_t* bytes) { + if (!enable()) { + return Status::InvalidArgument("cgroup is not enabled!"); + } + string cgroup_path; + RETURN_IF_ERROR(find_abs_cgroup_path("memory", &cgroup_path)); + string usage_file_path = cgroup_path + "/memory.usage_in_bytes"; + return read_cgroup_value(usage_file_path, bytes); +} + Status CGroupUtil::find_cgroup_cpu_limit(float* cpu_count) { if (!enable()) { return Status::InvalidArgument("cgroup is not enabled!"); diff --git a/be/src/util/cgroup_util.h b/be/src/util/cgroup_util.h index 286bf829391..015da392f7b 100644 --- a/be/src/util/cgroup_util.h +++ b/be/src/util/cgroup_util.h @@ -31,6 +31,10 @@ public: // set on any ancestor CGroups. static Status find_cgroup_mem_limit(int64_t* bytes); + // memory.usage_in_bytes ~= free.used + free.(buff/cache) - (buff) + // https://serverfault.com/questions/902009/the-memory-usage-reported-in-cgroup-differs-from-the-free-command + static Status find_cgroup_mem_usage(int64_t* bytes); + // Determines the CGroup cpu cores limit from the current processes' cgroup. static Status find_cgroup_cpu_limit(float* cpu_count); diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index 75c68cce4d1..ce42d69dccf 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -61,11 +61,9 @@ bvar::PassiveStatus<int64_t> g_proc_mem_no_allocator_cache( [](void*) { return MemInfo::proc_mem_no_allocator_cache(); }, nullptr); bool MemInfo::_s_initialized = false; -int64_t MemInfo::_s_physical_mem = -1; -int64_t MemInfo::_s_mem_limit = -1; -std::string MemInfo::_s_mem_limit_str = ""; -int64_t MemInfo::_s_soft_mem_limit = -1; -std::string MemInfo::_s_soft_mem_limit_str = ""; +std::atomic<int64_t> MemInfo::_s_physical_mem = std::numeric_limits<int64_t>::max(); +std::atomic<int64_t> MemInfo::_s_mem_limit = std::numeric_limits<int64_t>::max(); +std::atomic<int64_t> MemInfo::_s_soft_mem_limit = std::numeric_limits<int64_t>::max(); std::atomic<int64_t> MemInfo::_s_allocator_cache_mem = 0; std::string MemInfo::_s_allocator_cache_mem_str = ""; @@ -75,11 +73,10 @@ std::atomic<int64_t> MemInfo::refresh_interval_memory_growth = 0; static std::unordered_map<std::string, int64_t> _mem_info_bytes; std::atomic<int64_t> MemInfo::_s_sys_mem_available = -1; -std::string MemInfo::_s_sys_mem_available_str = ""; int64_t MemInfo::_s_sys_mem_available_low_water_mark = -1; int64_t MemInfo::_s_sys_mem_available_warning_water_mark = -1; -int64_t MemInfo::_s_process_minor_gc_size = -1; -int64_t MemInfo::_s_process_full_gc_size = -1; +std::atomic<int64_t> MemInfo::_s_process_minor_gc_size = -1; +std::atomic<int64_t> MemInfo::_s_process_full_gc_size = -1; std::mutex MemInfo::je_purge_dirty_pages_lock; std::condition_variable MemInfo::je_purge_dirty_pages_cv; std::atomic<bool> MemInfo::je_purge_dirty_pages_notify {false}; @@ -378,11 +375,13 @@ void MemInfo::refresh_proc_meminfo() { while (meminfo.good() && !meminfo.eof()) { getline(meminfo, line); std::vector<std::string> fields = strings::Split(line, " ", strings::SkipWhitespace()); - if (fields.size() < 2) continue; + if (fields.size() < 2) { + continue; + } std::string key = fields[0].substr(0, fields[0].size() - 1); StringParser::ParseResult result; - int64_t mem_value = + auto mem_value = StringParser::string_to_int<int64_t>(fields[1].data(), fields[1].size(), &result); if (result == StringParser::PARSE_SUCCESS) { @@ -393,58 +392,76 @@ void MemInfo::refresh_proc_meminfo() { } } } - if (meminfo.is_open()) meminfo.close(); - - if (_mem_info_bytes.find("MemAvailable") != _mem_info_bytes.end()) { - _s_sys_mem_available.store(_mem_info_bytes["MemAvailable"], std::memory_order_relaxed); - _s_sys_mem_available_str = PrettyPrinter::print( - _s_sys_mem_available.load(std::memory_order_relaxed), TUnit::BYTES); -#ifdef ADDRESS_SANITIZER - _s_sys_mem_available_str = - "[ASAN]" + - PrettyPrinter::print(_s_sys_mem_available.load(std::memory_order_relaxed), - TUnit::BYTES); -#else - _s_sys_mem_available_str = PrettyPrinter::print( - _s_sys_mem_available.load(std::memory_order_relaxed), TUnit::BYTES); -#endif + if (meminfo.is_open()) { + meminfo.close(); } -} -void MemInfo::init() { - refresh_proc_meminfo(); - _s_physical_mem = _mem_info_bytes["MemTotal"]; - - int64_t cgroup_mem_limit = 0; + // 1. calculate physical_mem + int64_t physical_mem = -1; + int64_t cgroup_mem_limit = -1; + physical_mem = _mem_info_bytes["MemTotal"]; Status status = CGroupUtil::find_cgroup_mem_limit(&cgroup_mem_limit); if (status.ok() && cgroup_mem_limit > 0) { - _s_physical_mem = std::min(_s_physical_mem, cgroup_mem_limit); + // In theory, always cgroup_mem_limit < physical_mem + physical_mem = std::min(physical_mem, cgroup_mem_limit); } - if (_s_physical_mem == -1) { - LOG(WARNING) << "Could not determine amount of physical memory on this machine."; + if (physical_mem <= 0) { + LOG(WARNING) + << "Could not determine amount of physical memory on this machine, physical_mem: " + << physical_mem; } - bool is_percent = true; - _s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent); - if (_s_mem_limit <= 0) { - LOG(WARNING) << "Failed to parse mem limit from '" + config::mem_limit + "'."; + // 2. if physical_mem changed, refresh mem limit and gc size. + if (physical_mem > 0 && _s_physical_mem.load(std::memory_order_relaxed) != physical_mem) { + _s_physical_mem.store(physical_mem); + + bool is_percent = true; + _s_mem_limit.store( + ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent)); + if (_s_mem_limit <= 0) { + LOG(WARNING) << "Failed to parse mem limit from '" + config::mem_limit + "'."; + } + if (_s_mem_limit > _s_physical_mem) { + LOG(WARNING) << "Memory limit " << PrettyPrinter::print(_s_mem_limit, TUnit::BYTES) + << " exceeds physical memory of " + << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) + << ". Using physical memory instead"; + _s_mem_limit.store(_s_physical_mem); + } + _s_soft_mem_limit.store(int64_t(_s_mem_limit * config::soft_mem_limit_frac)); + + _s_process_minor_gc_size.store(ParseUtil::parse_mem_spec(config::process_minor_gc_size, -1, + _s_mem_limit, &is_percent)); + _s_process_full_gc_size.store(ParseUtil::parse_mem_spec(config::process_full_gc_size, -1, + _s_mem_limit, &is_percent)); } - if (_s_mem_limit > _s_physical_mem) { - LOG(WARNING) << "Memory limit " << PrettyPrinter::print(_s_mem_limit, TUnit::BYTES) - << " exceeds physical memory of " - << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) - << ". Using physical memory instead"; - _s_mem_limit = _s_physical_mem; + + // 3. refresh process available memory + int64_t mem_available = -1; + int64_t cgroup_mem_usage = 0; + if (_mem_info_bytes.find("MemAvailable") != _mem_info_bytes.end()) { + mem_available = _mem_info_bytes["MemAvailable"]; } - _s_mem_limit_str = PrettyPrinter::print(_s_mem_limit, TUnit::BYTES); - _s_soft_mem_limit = int64_t(_s_mem_limit * config::soft_mem_limit_frac); - _s_soft_mem_limit_str = PrettyPrinter::print(_s_soft_mem_limit, TUnit::BYTES); + status = CGroupUtil::find_cgroup_mem_usage(&cgroup_mem_usage); + if (status.ok() && cgroup_mem_usage > 0 && cgroup_mem_limit > 0) { + if (mem_available < 0) { + mem_available = cgroup_mem_limit - cgroup_mem_usage; + } else { + mem_available = std::min(mem_available, cgroup_mem_limit - cgroup_mem_usage); + } + } + if (mem_available < 0) { + LOG(WARNING) << "Failed to get available memory, set MAX_INT."; + mem_available = std::numeric_limits<int64_t>::max(); + } + if (_s_sys_mem_available.load(std::memory_order_relaxed) != mem_available) { + _s_sys_mem_available.store(mem_available); + } +} - _s_process_minor_gc_size = - ParseUtil::parse_mem_spec(config::process_minor_gc_size, -1, _s_mem_limit, &is_percent); - _s_process_full_gc_size = - ParseUtil::parse_mem_spec(config::process_full_gc_size, -1, _s_mem_limit, &is_percent); +void MemInfo::init() { + refresh_proc_meminfo(); std::string line; int64_t _s_vm_min_free_kbytes = 0; @@ -506,8 +523,10 @@ void MemInfo::init() { << std::endl; } - LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) - << ", Mem Limit: " << _s_mem_limit_str + LOG(INFO) << "Physical Memory: " << _mem_info_bytes["MemTotal"] + << ", BE Available Physical Memory(consider cgroup): " + << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES) << ", Mem Limit: " + << PrettyPrinter::print(_s_mem_limit.load(std::memory_order_relaxed), TUnit::BYTES) << ", origin config value: " << config::mem_limit << ", System Mem Available Min Reserve: " << PrettyPrinter::print(_s_sys_mem_available_low_water_mark, TUnit::BYTES) @@ -528,9 +547,7 @@ void MemInfo::init() { bool is_percent = true; _s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent); - _s_mem_limit_str = PrettyPrinter::print(_s_mem_limit, TUnit::BYTES); _s_soft_mem_limit = static_cast<int64_t>(_s_mem_limit * config::soft_mem_limit_frac); - _s_soft_mem_limit_str = PrettyPrinter::print(_s_soft_mem_limit, TUnit::BYTES); LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES); _s_initialized = true; diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index c2bb8b209be..f6c9407d73e 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -41,6 +41,7 @@ #endif #include "common/config.h" #include "util/perf_counters.h" +#include "util/pretty_printer.h" namespace doris { @@ -67,7 +68,7 @@ public: // Get total physical memory in bytes (if has cgroups memory limits, return the limits). static inline int64_t physical_mem() { DCHECK(_s_initialized); - return _s_physical_mem; + return _s_physical_mem.load(std::memory_order_relaxed); } static void refresh_proc_meminfo(); @@ -76,7 +77,15 @@ public: return _s_sys_mem_available.load(std::memory_order_relaxed) - refresh_interval_memory_growth; } - static inline std::string sys_mem_available_str() { return _s_sys_mem_available_str; } + static inline std::string sys_mem_available_str() { +#ifdef ADDRESS_SANITIZER + return "[ASAN]" + PrettyPrinter::print(_s_sys_mem_available.load(std::memory_order_relaxed), + TUnit::BYTES); +#else + return PrettyPrinter::print(_s_sys_mem_available.load(std::memory_order_relaxed), + TUnit::BYTES); +#endif + } static inline int64_t sys_mem_available_low_water_mark() { return _s_sys_mem_available_low_water_mark; } @@ -169,19 +178,20 @@ public: static inline int64_t mem_limit() { DCHECK(_s_initialized); - return _s_mem_limit; + return _s_mem_limit.load(std::memory_order_relaxed); } static inline std::string mem_limit_str() { DCHECK(_s_initialized); - return _s_mem_limit_str; + return PrettyPrinter::print(_s_mem_limit.load(std::memory_order_relaxed), TUnit::BYTES); } static inline int64_t soft_mem_limit() { DCHECK(_s_initialized); - return _s_soft_mem_limit; + return _s_soft_mem_limit.load(std::memory_order_relaxed); } static inline std::string soft_mem_limit_str() { DCHECK(_s_initialized); - return _s_soft_mem_limit_str; + return PrettyPrinter::print(_s_soft_mem_limit.load(std::memory_order_relaxed), + TUnit::BYTES); } static bool is_exceed_soft_mem_limit(int64_t bytes = 0) { return proc_mem_no_allocator_cache() + bytes >= soft_mem_limit() || @@ -203,11 +213,9 @@ public: private: static bool _s_initialized; - static int64_t _s_physical_mem; - static int64_t _s_mem_limit; - static std::string _s_mem_limit_str; - static int64_t _s_soft_mem_limit; - static std::string _s_soft_mem_limit_str; + static std::atomic<int64_t> _s_physical_mem; + static std::atomic<int64_t> _s_mem_limit; + static std::atomic<int64_t> _s_soft_mem_limit; static std::atomic<int64_t> _s_allocator_cache_mem; static std::string _s_allocator_cache_mem_str; @@ -215,11 +223,10 @@ private: static std::atomic<int64_t> _s_proc_mem_no_allocator_cache; static std::atomic<int64_t> _s_sys_mem_available; - static std::string _s_sys_mem_available_str; static int64_t _s_sys_mem_available_low_water_mark; static int64_t _s_sys_mem_available_warning_water_mark; - static int64_t _s_process_minor_gc_size; - static int64_t _s_process_full_gc_size; + static std::atomic<int64_t> _s_process_minor_gc_size; + static std::atomic<int64_t> _s_process_full_gc_size; }; } // namespace doris --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
