This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new d3006ddd12 [branch-1.1-lts](cherry-pick) Some fixes for mem tracker
(#12889)
d3006ddd12 is described below
commit d3006ddd121f5c89dfea8f38f192de7d03fe5dd4
Author: Xinyi Zou <[email protected]>
AuthorDate: Thu Sep 22 21:47:45 2022 +0800
[branch-1.1-lts](cherry-pick) Some fixes for mem tracker (#12889)
* [fix][memtracker] remove gc and fix print
* [fix](memory) Fix BE OOM when load -238 fail
* [fix](memtracker) Process physical mem check does not include tc/jemalloc
allocator cache (#12688)
tcmalloc/jemalloc allocator cache does not participate in the mem check as
part of the process physical memory.
because new/malloc will trigger mem hook when using tcmalloc/jemalloc
allocator cache, but it may not actually alloc physical memory, which is not
expected in mem hook fail.
in addition:
The value of tcmalloc/jemalloc allocator cache is used as a mem tracker,
the parent is the process mem tracker, which is updated every 1s.
Modify the process default mem_limit to 90%. expect mem tracker to
effectively limit the memory usage of the process.
* Fix memory leak by calling in mem hook (#12708)
After the consume mem tracker exceeds the mem limit in the mem hook, the
boost stacktrace will be printed. A query/load will only be printed once, and
the process tracker will only be printed once per second.
After the process memory reaches the upper limit, the boost stacktrace will
be printed every second. The observed phenomena are as follows:
After query/load is canceled, the memory increases instantly;
tcmalloc profile total physical memory is less than perf process memory;
The process mem tracker is smaller than the perf process memory;
* [fix](memtracker) Fix thread mem tracker try consume accuracy #12782
* [Bugfix](mem) Fix memory limit check may overflow (#12776)
This bug is because the result of subtracting signed and unsigned numbers
may overflow if it is negative.
Co-authored-by: Zhengguo Yang <[email protected]>
---
be/src/common/config.h | 2 +-
be/src/common/daemon.cpp | 1 +
be/src/http/default_path_handlers.cpp | 5 +-
be/src/runtime/exec_env.h | 9 ++
be/src/runtime/exec_env_init.cpp | 1 +
be/src/runtime/load_channel.cpp | 9 +-
be/src/runtime/load_channel.h | 2 +-
be/src/runtime/load_channel_mgr.cpp | 10 +-
be/src/runtime/load_channel_mgr.h | 2 +-
be/src/runtime/memory/mem_tracker.cpp | 9 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 136 ++++++++---------------
be/src/runtime/memory/mem_tracker_limiter.h | 134 +++++++++++-----------
be/src/runtime/memory/thread_mem_tracker_mgr.cpp | 11 +-
be/src/runtime/memory/thread_mem_tracker_mgr.h | 18 +--
be/src/runtime/tablets_channel.cpp | 4 +
be/src/service/doris_main.cpp | 10 +-
be/src/util/mem_info.cpp | 15 ++-
be/src/util/mem_info.h | 37 +++++-
be/src/util/perf_counters.cpp | 6 +
be/src/util/perf_counters.h | 6 +-
be/src/util/system_metrics.cpp | 3 +-
21 files changed, 235 insertions(+), 195 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 7f1921d496..106609ee05 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -68,7 +68,7 @@ CONF_Int64(tc_max_total_thread_cache_bytes, "1073741824");
// defaults to bytes if no unit is given"
// must larger than 0. and if larger than physical memory size,
// it will be set to physical memory size.
-CONF_String(mem_limit, "80%");
+CONF_String(mem_limit, "90%");
// the port heartbeat service used
CONF_Int32(heartbeat_service_port, "9050");
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index ea628bb100..bb39bf13ef 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -68,6 +68,7 @@ namespace doris {
bool k_doris_exit = false;
void Daemon::tcmalloc_gc_thread() {
+ // TODO All cache GC wish to be supported
while
(!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(10))) {
size_t used_size = 0;
size_t free_size = 0;
diff --git a/be/src/http/default_path_handlers.cpp
b/be/src/http/default_path_handlers.cpp
index c7cdcd2ad8..3efed02a09 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -32,6 +32,7 @@
#include "runtime/mem_tracker.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "util/debug_util.h"
+#include "util/perf_counters.h"
#include "util/pretty_printer.h"
#include "util/thread.h"
@@ -88,8 +89,8 @@ void mem_usage_handler(const std::shared_ptr<MemTracker>&
mem_tracker,
(*output) << "<pre>"
<< "Mem Limit: " <<
PrettyPrinter::print(mem_tracker->limit(), TUnit::BYTES)
<< std::endl
- << "Mem Consumption: "
- << PrettyPrinter::print(mem_tracker->consumption(),
TUnit::BYTES) << std::endl
+ << "Physical Mem From Perf: "
+ << PrettyPrinter::print(PerfCounters::get_vm_rss(),
TUnit::BYTES) << std::endl
<< "</pre>";
} else {
(*output) << "<pre>"
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index e73eaa1c1f..a28e497a6b 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -41,6 +41,7 @@ class EvHttpServer;
class ExternalScanContextMgr;
class FragmentMgr;
class ResultCache;
+class NewMemTracker;
class LoadPathMgr;
class LoadStreamMgr;
class MemTracker;
@@ -123,6 +124,13 @@ public:
std::shared_ptr<MemTrackerLimiter> new_process_mem_tracker() { return
_process_mem_tracker; }
MemTrackerLimiter* process_mem_tracker_raw() { return
_process_mem_tracker_raw; }
+ void set_process_mem_tracker(const std::shared_ptr<MemTrackerLimiter>&
tracker) {
+ _process_mem_tracker = tracker;
+ _process_mem_tracker_raw = tracker.get();
+ }
+ std::shared_ptr<NewMemTracker> allocator_cache_mem_tracker() {
+ return _allocator_cache_mem_tracker;
+ }
std::shared_ptr<MemTrackerLimiter> query_pool_mem_tracker() { return
_query_pool_mem_tracker; }
std::shared_ptr<MemTrackerLimiter> load_pool_mem_tracker() { return
_load_pool_mem_tracker; }
MemTrackerTaskPool* task_pool_mem_tracker_registry() { return
_task_pool_mem_tracker_registry; }
@@ -199,6 +207,7 @@ private:
// The ancestor for all trackers. Every tracker is visible from the
process down.
// Not limit total memory by process tracker, and it's just used to track
virtual memory of process.
std::shared_ptr<MemTrackerLimiter> _process_mem_tracker;
+ std::shared_ptr<NewMemTracker> _allocator_cache_mem_tracker;
MemTrackerLimiter* _process_mem_tracker_raw;
// The ancestor for all querys tracker.
std::shared_ptr<MemTrackerLimiter> _query_pool_mem_tracker;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 54640a6e70..0fe31a1a6b 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -215,6 +215,7 @@ Status ExecEnv::_init_mem_tracker() {
}
#endif
+ _allocator_cache_mem_tracker =
std::make_shared<NewMemTracker>("Tc/JemallocAllocatorCache");
_query_pool_mem_tracker =
std::make_shared<MemTrackerLimiter>(-1, "QueryPool",
_process_mem_tracker);
_load_pool_mem_tracker =
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index 715762bd18..506472ba0a 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -86,7 +86,7 @@ Status LoadChannel::add_batch(const
PTabletWriterAddBatchRequest& request,
}
// 2. check if mem consumption exceed limit
- handle_mem_exceed_limit(false);
+ RETURN_IF_ERROR(handle_mem_exceed_limit(false));
// 3. add batch to tablets channel
if (request.has_row_batch()) {
@@ -111,11 +111,11 @@ Status LoadChannel::add_batch(const
PTabletWriterAddBatchRequest& request,
return st;
}
-void LoadChannel::handle_mem_exceed_limit(bool force) {
+Status LoadChannel::handle_mem_exceed_limit(bool force) {
// lock so that only one thread can check mem limit
std::lock_guard<std::mutex> l(_lock);
if (!(force || _mem_tracker->limit_exceeded())) {
- return;
+ return Status::OK();
}
if (!force) {
@@ -125,12 +125,13 @@ void LoadChannel::handle_mem_exceed_limit(bool force) {
std::shared_ptr<TabletsChannel> channel;
if (_find_largest_consumption_channel(&channel)) {
- channel->reduce_mem_usage(_mem_tracker->limit());
+ return channel->reduce_mem_usage(_mem_tracker->limit());
} else {
// should not happen, add log to observe
LOG(WARNING) << "fail to find suitable tablets-channel when memory
exceed. "
<< "load_id=" << _load_id;
}
+ return Status::OK();
}
// lock should be held when calling this method
diff --git a/be/src/runtime/load_channel.h b/be/src/runtime/load_channel.h
index 13490f5fa8..c74718c8ab 100644
--- a/be/src/runtime/load_channel.h
+++ b/be/src/runtime/load_channel.h
@@ -63,7 +63,7 @@ public:
// If yes, it will pick a tablets channel to try to reduce memory
consumption.
// If force is true, even if this load channel does not exceeds limit, it
will still
// try to reduce memory.
- void handle_mem_exceed_limit(bool force);
+ Status handle_mem_exceed_limit(bool force);
int64_t mem_consumption() const { return _mem_tracker->consumption(); }
diff --git a/be/src/runtime/load_channel_mgr.cpp
b/be/src/runtime/load_channel_mgr.cpp
index 98ef8c724d..a76109d700 100644
--- a/be/src/runtime/load_channel_mgr.cpp
+++ b/be/src/runtime/load_channel_mgr.cpp
@@ -152,7 +152,7 @@ Status LoadChannelMgr::add_batch(const
PTabletWriterAddBatchRequest& request,
// 2. check if mem consumption exceed limit
// If this is a high priority load task, do not handle this.
// because this may block for a while, which may lead to rpc timeout.
- _handle_mem_exceed_limit();
+ RETURN_IF_ERROR(_handle_mem_exceed_limit());
}
// 3. add batch to load channel
@@ -175,11 +175,11 @@ Status LoadChannelMgr::add_batch(const
PTabletWriterAddBatchRequest& request,
return Status::OK();
}
-void LoadChannelMgr::_handle_mem_exceed_limit() {
+Status LoadChannelMgr::_handle_mem_exceed_limit() {
// lock so that only one thread can check mem limit
std::lock_guard<std::mutex> l(_lock);
if (!_mem_tracker->limit_exceeded()) {
- return;
+ return Status::OK();
}
int64_t max_consume = 0;
@@ -198,14 +198,14 @@ void LoadChannelMgr::_handle_mem_exceed_limit() {
if (max_consume == 0) {
// should not happen, add log to observe
LOG(WARNING) << "failed to find suitable load channel when total load
mem limit exceed";
- return;
+ return Status::OK();
}
DCHECK(channel.get() != nullptr);
// force reduce mem limit of the selected channel
LOG(INFO) << "reducing memory of " << *channel << " because total load mem
consumption "
<< _mem_tracker->consumption() << " has exceeded limit " <<
_mem_tracker->limit();
- channel->handle_mem_exceed_limit(true);
+ return channel->handle_mem_exceed_limit(true);
}
Status LoadChannelMgr::cancel(const PTabletWriterCancelRequest& params) {
diff --git a/be/src/runtime/load_channel_mgr.h
b/be/src/runtime/load_channel_mgr.h
index 1da0ec75a5..a522ab7d39 100644
--- a/be/src/runtime/load_channel_mgr.h
+++ b/be/src/runtime/load_channel_mgr.h
@@ -59,7 +59,7 @@ public:
private:
// check if the total load mem consumption exceeds limit.
// If yes, it will pick a load channel to try to reduce memory consumption.
- void _handle_mem_exceed_limit();
+ Status _handle_mem_exceed_limit();
Status _start_bg_worker();
diff --git a/be/src/runtime/memory/mem_tracker.cpp
b/be/src/runtime/memory/mem_tracker.cpp
index 8c0ae6ebba..07ca883a94 100644
--- a/be/src/runtime/memory/mem_tracker.cpp
+++ b/be/src/runtime/memory/mem_tracker.cpp
@@ -105,10 +105,11 @@ void
NewMemTracker::make_group_snapshot(std::vector<NewMemTracker::Snapshot>* sn
}
std::string NewMemTracker::log_usage(NewMemTracker::Snapshot snapshot) {
- return fmt::format("NewMemTracker Label={}, Parent Label={}, Used={},
Peak={}", snapshot.label,
- snapshot.parent,
- PrettyPrinter::print(snapshot.cur_consumption,
TUnit::BYTES),
- PrettyPrinter::print(snapshot.peak_consumption,
TUnit::BYTES));
+ return fmt::format(
+ "MemTracker Label={}, Parent Label={}, Used={}({} B), Peak={}({}
B)", snapshot.label,
+ snapshot.parent, PrettyPrinter::print(snapshot.cur_consumption,
TUnit::BYTES),
+ snapshot.cur_consumption,
PrettyPrinter::print(snapshot.peak_consumption, TUnit::BYTES),
+ snapshot.peak_consumption);
}
} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index fd4c46259c..0494657396 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -24,7 +24,6 @@
#include "gutil/once.h"
#include "runtime/runtime_state.h"
#include "runtime/thread_context.h"
-#include "service/backend_options.h"
#include "util/pretty_printer.h"
#include "util/string_util.h"
@@ -49,6 +48,8 @@ MemTrackerLimiter::MemTrackerLimiter(int64_t byte_limit,
const std::string& labe
MemTrackerLimiter* tracker = this;
while (tracker != nullptr) {
_all_ancestors.push_back(tracker);
+ // Process tracker does not participate in the process memory limit,
process tracker consumption is virtual memory,
+ // and there is a diff between the real physical memory value of the
process. It is replaced by check_sys_mem_info.
if (tracker->has_limit() && tracker->label() != "Process")
_limited_ancestors.push_back(tracker);
tracker = tracker->_parent.get();
@@ -123,42 +124,6 @@ int64_t MemTrackerLimiter::get_lowest_limit() const {
return min_limit;
}
-bool MemTrackerLimiter::gc_memory(int64_t max_consumption) {
- if (max_consumption < 0) return true;
- std::lock_guard<std::mutex> l(_gc_lock);
- int64_t pre_gc_consumption = consumption();
- // Check if someone gc'd before us
- if (pre_gc_consumption < max_consumption) return false;
-
- int64_t curr_consumption = pre_gc_consumption;
- // Free some extra memory to avoid frequent GC, 4M is an empirical value,
maybe it will be tested later.
- const int64_t EXTRA_BYTES_TO_FREE = 4L * 1024L * 1024L * 1024L;
- // Try to free up some memory
- for (int i = 0; i < _gc_functions.size(); ++i) {
- // Try to free up the amount we are over plus some extra so that we
don't have to
- // immediately GC again. Don't free all the memory since that can be
unnecessarily
- // expensive.
- int64_t bytes_to_free = curr_consumption - max_consumption +
EXTRA_BYTES_TO_FREE;
- _gc_functions[i](bytes_to_free);
- curr_consumption = consumption();
- if (max_consumption - curr_consumption <= EXTRA_BYTES_TO_FREE) break;
- }
-
- return curr_consumption > max_consumption;
-}
-
-Status MemTrackerLimiter::try_gc_memory(int64_t bytes) {
- if (UNLIKELY(gc_memory(_limit - bytes))) {
- return Status::MemoryLimitExceeded(fmt::format(
- "failed_alloc_size={} B, exceeded_tracker={}, limit={} B,
peak_used={} B, "
- "current_used={} B",
- bytes, label(), _limit, _consumption->value(),
_consumption->current_value()));
- }
- VLOG_NOTICE << "GC succeeded, TryConsume bytes=" << bytes
- << " consumption=" << _consumption->current_value() << "
limit=" << _limit;
- return Status::OK();
-}
-
// Calling this on the query tracker results in output like:
//
// Query(4a4c81fedaed337d:4acadfda00000000) Limit=10.00 GB Total=508.28 MB
Peak=508.45 MB
@@ -185,10 +150,10 @@ std::string MemTrackerLimiter::log_usage(int
max_recursive_depth, int64_t* logge
int64_t peak_consumption = _consumption->value();
if (logged_consumption != nullptr) *logged_consumption = curr_consumption;
- std::string detail = "MemTrackerLimiter Label={}, Limit={}, Used={},
Peak={}, Exceeded={}";
- detail = fmt::format(detail, _label, PrettyPrinter::print(_limit,
TUnit::BYTES),
- PrettyPrinter::print(curr_consumption, TUnit::BYTES),
- PrettyPrinter::print(peak_consumption, TUnit::BYTES),
+ std::string detail =
+ "MemTrackerLimiter Label={}, Limit={}({} B), Used={}({} B),
Peak={}({} B), Exceeded={}";
+ detail = fmt::format(detail, _label, print_bytes(_limit), _limit,
print_bytes(curr_consumption),
+ curr_consumption, print_bytes(peak_consumption),
peak_consumption,
limit_exceeded() ? "true" : "false");
// This call does not need the children, so return early.
@@ -222,41 +187,38 @@ std::string MemTrackerLimiter::log_usage(int
max_recursive_depth,
if (!usage_string.empty()) usage_strings.push_back(usage_string);
*logged_consumption += tracker_consumption;
}
- return join(usage_strings, "\n");
-}
-
-Status MemTrackerLimiter::mem_limit_exceeded_construct(const std::string& msg)
{
- std::string detail = fmt::format(
- "{}, backend {} process memory used {}, process limit {}. If is
query, can "
- "change the limit "
- "by `set exec_mem_limit=xxx`, details mem usage see be.INFO.",
- msg, BackendOptions::get_localhost(),
- PrettyPrinter::print(PerfCounters::get_vm_rss(), TUnit::BYTES),
- PrettyPrinter::print(MemInfo::mem_limit(), TUnit::BYTES));
- return Status::MemoryLimitExceeded(detail);
+ return usage_strings.size() == 0 ? "" : "\n " + join(usage_strings, "\n
");
}
void MemTrackerLimiter::print_log_usage(const std::string& msg) {
// only print the tracker log_usage in be log.
std::string detail = msg;
+ detail += "\n " + fmt::format(
+ "process memory used {}, limit {}, hard limit
{}, tc/jemalloc "
+ "allocator cache {}",
+ PerfCounters::get_vm_rss_str(),
MemInfo::mem_limit_str(),
+ print_bytes(MemInfo::hard_mem_limit()),
+ MemInfo::allocator_cache_mem_str());
if (_print_log_usage) {
if (_label == "Process") {
// Dumping the process MemTracker is expensive. Limiting the
recursive depth to two
// levels limits the level of detail to a one-line summary for
each query MemTracker.
- detail += "\n" + log_usage(2);
+ detail += "\n " + log_usage(2);
} else {
- detail += "\n" + log_usage();
+ detail += "\n " + log_usage();
}
- detail += "\n" +
boost::stacktrace::to_string(boost::stacktrace::stacktrace());
+ // TODO: memory leak by calling `boost::stacktrace` in tcmalloc hook,
+ // test whether overwriting malloc/free is the same problem in
jemalloc/tcmalloc.
+ // detail += "\n" +
boost::stacktrace::to_string(boost::stacktrace::stacktrace());
LOG(WARNING) << detail;
_print_log_usage = false;
}
}
-Status MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
- int64_t failed_allocation_size) {
+std::string MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
+ int64_t
failed_allocation_size) {
STOP_CHECK_THREAD_MEM_TRACKER_LIMIT();
- std::string detail = fmt::format("Memory limit
exceeded:<consuming_tracker={}, ", _label);
+ std::string detail = fmt::format("Memory limit exceeded:<consuming
tracker:<{}>, ", _label);
MemTrackerLimiter* exceeded_tracker = nullptr;
MemTrackerLimiter* max_consumption_tracker = nullptr;
int64_t free_size = INT64_MAX;
@@ -275,57 +237,51 @@ Status MemTrackerLimiter::mem_limit_exceeded(const
std::string& msg,
}
}
- auto sys_exceed_st = check_sys_mem_info(failed_allocation_size);
MemTrackerLimiter* print_log_usage_tracker = nullptr;
if (exceeded_tracker != nullptr) {
- detail += fmt::format(
- "failed_alloc_size={} B, exceeded_tracker={}, limit={} B,
peak_used={} B, "
- "current_used={} B>, executing_msg:<{}>",
- PrettyPrinter::print(failed_allocation_size, TUnit::BYTES),
- exceeded_tracker->label(), exceeded_tracker->limit(),
- exceeded_tracker->peak_consumption(),
exceeded_tracker->consumption(), msg);
+ detail += limit_exceeded_errmsg_prefix_str(failed_allocation_size,
exceeded_tracker);
print_log_usage_tracker = exceeded_tracker;
- } else if (!sys_exceed_st) {
- detail += fmt::format("{}>, executing_msg:<{}>",
sys_exceed_st.get_error_msg(), msg);
+ } else if (sys_mem_exceed_limit_check(failed_allocation_size)) {
+ detail += fmt::format("{}>, executing msg:<{}>",
+
limit_exceeded_errmsg_sys_str(failed_allocation_size), msg);
} else if (max_consumption_tracker != nullptr) {
// must after check_sys_mem_info false
detail += fmt::format(
- "failed_alloc_size={} B, max_consumption_tracker={}, limit={}
B, peak_used={} B, "
- "current_used={} B>, executing_msg:<{}>",
- PrettyPrinter::print(failed_allocation_size, TUnit::BYTES),
- max_consumption_tracker->label(),
max_consumption_tracker->limit(),
- max_consumption_tracker->peak_consumption(),
max_consumption_tracker->consumption(),
- msg);
+ "failed alloc size {}, max consumption tracker:<{}>, limit {},
peak used {}, "
+ "current used {}>, executing msg:<{}>",
+ print_bytes(failed_allocation_size),
max_consumption_tracker->label(),
+ print_bytes(max_consumption_tracker->limit()),
+ print_bytes(max_consumption_tracker->peak_consumption()),
+ print_bytes(max_consumption_tracker->consumption()), msg);
print_log_usage_tracker = max_consumption_tracker;
} else {
// The limit of the current tracker and parents is less than 0, the
consume will not fail,
// and the current process memory has no excess limit.
- detail += fmt::format("unknown exceed reason, executing_msg:<{}>",
msg);
+ detail += fmt::format("unknown exceed reason, executing msg:<{}>",
msg);
print_log_usage_tracker =
ExecEnv::GetInstance()->process_mem_tracker_raw();
}
- auto st = MemTrackerLimiter::mem_limit_exceeded_construct(detail);
- if (print_log_usage_tracker != nullptr)
- print_log_usage_tracker->print_log_usage(st.get_error_msg());
- return st;
+ auto failed_msg =
MemTrackerLimiter::limit_exceeded_errmsg_suffix_str(detail);
+ if (print_log_usage_tracker != nullptr)
print_log_usage_tracker->print_log_usage(failed_msg);
+ return failed_msg;
}
-Status MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
- MemTrackerLimiter* failed_tracker,
- Status failed_try_consume_st) {
+std::string MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
+ MemTrackerLimiter*
failed_tracker,
+ const std::string&
limit_exceeded_errmsg_prefix) {
STOP_CHECK_THREAD_MEM_TRACKER_LIMIT();
std::string detail =
- fmt::format("Memory limit exceeded:<consuming_tracker={}, {}>,
executing_msg:<{}>",
- _label, failed_try_consume_st.get_error_msg(), msg);
- auto st = MemTrackerLimiter::mem_limit_exceeded_construct(detail);
- failed_tracker->print_log_usage(st.get_error_msg());
- return st;
+ fmt::format("Memory limit exceeded:<consuming tracker:<{}>, {}>,
executing msg:<{}>",
+ _label, limit_exceeded_errmsg_prefix, msg);
+ auto failed_msg =
MemTrackerLimiter::limit_exceeded_errmsg_suffix_str(detail);
+ failed_tracker->print_log_usage(failed_msg);
+ return failed_msg;
}
Status MemTrackerLimiter::mem_limit_exceeded(RuntimeState* state, const
std::string& msg,
int64_t failed_alloc_size) {
- Status rt = mem_limit_exceeded(msg, failed_alloc_size);
- state->log_error(rt.to_string());
- return rt;
+ auto failed_msg = mem_limit_exceeded(msg, failed_alloc_size);
+ state->log_error(failed_msg);
+ return Status::MemoryLimitExceeded(failed_msg);
}
} // namespace doris
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index c2541b6aea..0886279c6e 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -22,8 +22,10 @@
#include "common/config.h"
#include "runtime/exec_env.h"
#include "runtime/memory/mem_tracker.h"
+#include "service/backend_options.h"
#include "util/mem_info.h"
#include "util/perf_counters.h"
+#include "util/pretty_printer.h"
namespace doris {
@@ -64,19 +66,23 @@ public:
size_t upper_level) const;
public:
- static Status check_sys_mem_info(int64_t bytes) {
+ static bool sys_mem_exceed_limit_check(int64_t bytes) {
// Limit process memory usage using the actual physical memory of the
process in `/proc/self/status`.
// This is independent of the consumption value of the mem tracker,
which counts the virtual memory
// of the process malloc.
// for fast, expect MemInfo::initialized() to be true.
- if (PerfCounters::get_vm_rss() + bytes >= MemInfo::mem_limit()) {
- auto st = Status::MemoryLimitExceeded(
- fmt::format("process memory used {} B, exceed limit {} B,
failed_alloc_size={} B",
- PerfCounters::get_vm_rss(), MemInfo::mem_limit(), bytes));
-
ExecEnv::GetInstance()->process_mem_tracker_raw()->print_log_usage(st.get_error_msg());
- return st;
+ // tcmalloc/jemalloc allocator cache does not participate in the mem
check as part of the process physical memory.
+ // because `new/malloc` will trigger mem hook when using
tcmalloc/jemalloc allocator cache,
+ // but it may not actually alloc physical memory, which is not
expected in mem hook fail.
+ //
+ // TODO: In order to ensure no OOM, currently reserve 200M, and then
use the free mem in /proc/meminfo to ensure no OOM.
+ if (PerfCounters::get_vm_rss() -
static_cast<int64_t>(MemInfo::allocator_cache_mem()) +
+ bytes >=
+ MemInfo::mem_limit() ||
+ PerfCounters::get_vm_rss() + bytes >= MemInfo::hard_mem_limit()) {
+ return true;
}
- return Status::OK();
+ return false;
}
int64_t group_num() const { return _group_num; }
@@ -108,21 +114,6 @@ public:
// Returns the lowest limit for this tracker limiter and its ancestors.
Returns -1 if there is no limit.
int64_t get_lowest_limit() const;
- typedef std::function<void(int64_t bytes_to_free)> GcFunction;
- // Add a function 'f' to be called if the limit is reached, if none of the
other
- // previously-added GC functions were successful at freeing up enough
memory.
- // 'f' does not need to be thread-safe as long as it is added to only one
tracker limiter.
- // Note that 'f' must be valid for the lifetime of this tracker limiter.
- void add_gc_function(GcFunction f) { _gc_functions.push_back(f); }
-
- // TODO Should be managed in a separate process_mem_mgr, not in
NewMemTracker
- // If consumption is higher than max_consumption, attempts to free memory
by calling
- // any added GC functions. Returns true if max_consumption is still
exceeded. Takes gc_lock.
- // Note: If the cache of segment/chunk is released due to insufficient
query memory at a certain moment,
- // the performance of subsequent queries may be degraded, so the use of gc
function should be careful enough.
- bool gc_memory(int64_t max_consumption);
- Status try_gc_memory(int64_t bytes);
-
public:
// up to (but not including) end_tracker.
// This happens when we want to update tracking on a particular mem
tracker but the consumption
@@ -147,15 +138,16 @@ public:
// Limiting the recursive depth reduces the cost of dumping, particularly
// for the process tracker limiter.
std::string log_usage(int max_recursive_depth = INT_MAX, int64_t*
logged_consumption = nullptr);
+ void print_log_usage(const std::string& msg);
// Log the memory usage when memory limit is exceeded and return a status
object with
// msg of the allocation which caused the limit to be exceeded.
// If 'failed_allocation_size' is greater than zero, logs the allocation
size. If
// 'failed_allocation_size' is zero, nothing about the allocation size is
logged.
// If 'state' is non-nullptr, logs the error to 'state'.
- Status mem_limit_exceeded(const std::string& msg, int64_t
failed_allocation_size = 0);
- Status mem_limit_exceeded(const std::string& msg, MemTrackerLimiter*
failed_tracker,
- Status failed_try_consume_st);
+ std::string mem_limit_exceeded(const std::string& msg, int64_t
failed_allocation_size = 0);
+ std::string mem_limit_exceeded(const std::string& msg, MemTrackerLimiter*
failed_tracker,
+ const std::string&
limit_exceeded_errmsg_prefix);
Status mem_limit_exceeded(RuntimeState* state, const std::string& msg,
int64_t failed_allocation_size = 0);
@@ -169,6 +161,10 @@ public:
return msg.str();
}
+ static std::string print_bytes(int64_t bytes) {
+ return PrettyPrinter::print(bytes, TUnit::BYTES);
+ }
+
private:
// The following func, for automatic memory tracking and limiting based on
system memory allocation.
friend class ThreadMemTrackerMgr;
@@ -183,7 +179,7 @@ private:
// they can all consume 'bytes' without exceeding limit. If limit would be
exceed,
// no MemTrackerLimiters are updated. Returns true if the consumption was
successfully updated.
WARN_UNUSED_RESULT
- Status try_consume(int64_t bytes);
+ bool try_consume(int64_t bytes, std::string& failed_msg);
// When the accumulated untracked memory value exceeds the upper limit,
// the current value is returned and set to 0.
@@ -197,8 +193,33 @@ private:
const std::list<MemTrackerLimiter*>& trackers,
int64_t* logged_consumption);
- static Status mem_limit_exceeded_construct(const std::string& msg);
- void print_log_usage(const std::string& msg);
+ static std::string limit_exceeded_errmsg_prefix_str(int64_t bytes,
+ MemTrackerLimiter*
exceed_tracker) {
+ return fmt::format(
+ "failed alloc size {}, exceeded tracker:<{}>, limit {}, peak "
+ "used {}, current used {}",
+ print_bytes(bytes), exceed_tracker->label(),
print_bytes(exceed_tracker->limit()),
+ print_bytes(exceed_tracker->_consumption->value()),
+ print_bytes(exceed_tracker->_consumption->current_value()));
+ }
+
+ static std::string limit_exceeded_errmsg_suffix_str(const std::string&
msg) {
+ return fmt::format(
+ "{}. backend {} process memory used {}, limit {}. If query
tracker exceed, `set "
+ "exec_mem_limit=8G` to change limit, details mem usage see
be.INFO.",
+ msg, BackendOptions::get_localhost(),
PerfCounters::get_vm_rss_str(),
+ MemInfo::mem_limit_str());
+ }
+
+ static std::string limit_exceeded_errmsg_sys_str(int64_t bytes) {
+ auto err_msg = fmt::format(
+ "process memory used {}, tc/jemalloc allocator cache {},
exceed limit {}, failed "
+ "alloc size {}",
+ PerfCounters::get_vm_rss_str(),
MemInfo::allocator_cache_mem_str(),
+ MemInfo::mem_limit_str(), print_bytes(bytes));
+
ExecEnv::GetInstance()->process_mem_tracker_raw()->print_log_usage(err_msg);
+ return err_msg;
+ }
private:
// Limit on memory consumption, in bytes. If limit_ == -1, there is no
consumption limit. Used in log_usage。
@@ -230,19 +251,6 @@ private:
std::atomic_size_t _had_child_count = 0;
bool _print_log_usage = false;
-
- // Lock to protect gc_memory(). This prevents many GCs from occurring at
once.
- std::mutex _gc_lock;
- // Functions to call after the limit is reached to free memory.
- // GcFunctions can be attached to a NewMemTracker in order to free up
memory if the limit is
- // reached. If limit_exceeded() is called and the limit is exceeded, it
will first call
- // the GcFunctions to try to free memory and recheck the limit. For
example, the process
- // tracker has a GcFunction that releases any unused memory still held by
tcmalloc, so
- // this will be called before the process limit is reported as exceeded.
GcFunctions are
- // called in the order they are added, so expensive functions should be
added last.
- // GcFunctions are called with a global lock held, so should be
non-blocking and not
- // call back into MemTrackers, except to release memory.
- std::vector<GcFunction> _gc_functions;
};
inline void MemTrackerLimiter::consume(int64_t bytes) {
@@ -271,12 +279,16 @@ inline void
MemTrackerLimiter::cache_consume_local(int64_t bytes) {
}
}
-inline Status MemTrackerLimiter::try_consume(int64_t bytes) {
+inline bool MemTrackerLimiter::try_consume(int64_t bytes, std::string&
failed_msg) {
if (bytes <= 0) {
release(-bytes);
- return Status::OK();
+ failed_msg = std::string();
+ return true;
+ }
+ if (sys_mem_exceed_limit_check(bytes)) {
+ failed_msg = limit_exceeded_errmsg_sys_str(bytes);
+ return false;
}
- RETURN_IF_ERROR(check_sys_mem_info(bytes));
int i;
// Walk the tracker tree top-down.
for (i = _all_ancestors.size() - 1; i >= 0; --i) {
@@ -286,39 +298,33 @@ inline Status MemTrackerLimiter::try_consume(int64_t
bytes) {
if (tracker->limit() < 0 || tracker->label() == "Process") {
tracker->_consumption->add(bytes); // No limit at this tracker.
} else {
- // If TryConsume fails, we can try to GC, but we may need to try
several times if
- // there are concurrent consumers because we don't take a lock
before trying to
- // update _consumption.
- while (true) {
- if (LIKELY(tracker->_consumption->try_add(bytes,
tracker->limit()))) break;
- Status st = tracker->try_gc_memory(bytes);
- if (!st) {
- // Failed for this mem tracker. Roll back the ones that
succeeded.
- for (int j = _all_ancestors.size() - 1; j > i; --j) {
- _all_ancestors[j]->_consumption->add(-bytes);
- }
- return st;
+ if (!tracker->_consumption->try_add(bytes, tracker->limit())) {
+ // Failed for this mem tracker. Roll back the ones that
succeeded.
+ for (int j = _all_ancestors.size() - 1; j > i; --j) {
+ _all_ancestors[j]->_consumption->add(-bytes);
}
+ failed_msg = limit_exceeded_errmsg_prefix_str(bytes, tracker);
+ return false;
}
}
}
// Everyone succeeded, return.
DCHECK_EQ(i, -1);
- return Status::OK();
+ failed_msg = std::string();
+ return true;
}
inline Status MemTrackerLimiter::check_limit(int64_t bytes) {
if (bytes <= 0) return Status::OK();
- RETURN_IF_ERROR(check_sys_mem_info(bytes));
+ if (sys_mem_exceed_limit_check(bytes)) {
+ return
Status::MemoryLimitExceeded(limit_exceeded_errmsg_sys_str(bytes));
+ }
int i;
// Walk the tracker tree top-down.
for (i = _limited_ancestors.size() - 1; i >= 0; --i) {
MemTrackerLimiter* tracker = _limited_ancestors[i];
- // Process tracker does not participate in the process memory limit,
process tracker consumption is virtual memory,
- // and there is a diff between the real physical memory value of the
process. It is replaced by check_sys_mem_info.
- while (true) {
- if (LIKELY(tracker->_consumption->current_value() + bytes <
tracker->limit())) break;
- RETURN_IF_ERROR(tracker->try_gc_memory(bytes));
+ if (tracker->_consumption->current_value() + bytes > tracker->limit())
{
+ return
Status::MemoryLimitExceeded(limit_exceeded_errmsg_prefix_str(bytes, tracker));
}
}
return Status::OK();
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
index 30f7e7f10b..40879b6c4a 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
@@ -51,16 +51,17 @@ void ThreadMemTrackerMgr::exceeded_cancel_task(const
std::string& cancel_details
}
}
-void ThreadMemTrackerMgr::exceeded(Status failed_try_consume_st) {
+void ThreadMemTrackerMgr::exceeded(const std::string& failed_msg) {
if (_cb_func != nullptr) {
_cb_func();
}
if (is_attach_query()) {
- auto st = _limiter_tracker_raw->mem_limit_exceeded(fmt::format("exec
node:<{}>", ""),
-
_limiter_tracker->parent().get(),
- failed_try_consume_st);
- exceeded_cancel_task(st.get_error_msg());
+ auto cancel_msg = _limiter_tracker_raw->mem_limit_exceeded(
+ fmt::format("exec node:<{}>", ""),
+ _limiter_tracker_raw->parent().get(), failed_msg);
+ exceeded_cancel_task(cancel_msg);
_check_limit = false; // Make sure it will only be canceled once
}
}
+
} // namespace doris
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h
b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index 5135d6222b..3f652ecd5a 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -103,12 +103,14 @@ private:
// If tryConsume fails due to task mem tracker exceeding the limit, the
task must be canceled
void exceeded_cancel_task(const std::string& cancel_details);
- void exceeded(Status failed_try_consume_st);
+ void exceeded(const std::string& failed_msg);
private:
// Cache untracked mem, only update to _untracked_mems when switching mem
tracker.
// Frequent calls to unordered_map _untracked_mems[] in consume will
degrade performance.
int64_t _untracked_mem = 0;
+ int64_t old_untracked_mem = 0;
+ std::string failed_msg = std::string();
std::shared_ptr<MemTrackerLimiter> _limiter_tracker;
std::vector<NewMemTracker*> _consumer_tracker_stack;
@@ -167,6 +169,7 @@ inline void ThreadMemTrackerMgr::flush_untracked_mem() {
// Temporary memory may be allocated during the consumption of the mem
tracker, which will lead to entering
// the TCMalloc Hook again, so suspend consumption to avoid falling into
an infinite loop.
_stop_consume = true;
+ old_untracked_mem = _untracked_mem;
DCHECK(_limiter_tracker);
if (CheckLimit) {
#ifndef BE_TEST
@@ -179,20 +182,19 @@ inline void ThreadMemTrackerMgr::flush_untracked_mem() {
// DCHECK(!_check_attach || btls_key != EMPTY_BTLS_KEY ||
// _limiter_tracker->label() != "Process");
#endif
- Status st = _limiter_tracker_raw->try_consume(_untracked_mem);
- if (!st) {
+ if (!_limiter_tracker_raw->try_consume(old_untracked_mem, failed_msg))
{
// The memory has been allocated, so when TryConsume fails, need
to continue to complete
// the consume to ensure the accuracy of the statistics.
- _limiter_tracker_raw->consume(_untracked_mem);
- exceeded(st);
+ _limiter_tracker_raw->consume(old_untracked_mem);
+ exceeded(failed_msg);
}
} else {
- _limiter_tracker_raw->consume(_untracked_mem);
+ _limiter_tracker_raw->consume(old_untracked_mem);
}
for (auto tracker : _consumer_tracker_stack) {
- tracker->consume(_untracked_mem);
+ tracker->consume(old_untracked_mem);
}
- _untracked_mem = 0;
+ _untracked_mem -= old_untracked_mem;
_stop_consume = false;
}
diff --git a/be/src/runtime/tablets_channel.cpp
b/be/src/runtime/tablets_channel.cpp
index 1e1ab2cd7b..3785eedd10 100644
--- a/be/src/runtime/tablets_channel.cpp
+++ b/be/src/runtime/tablets_channel.cpp
@@ -272,6 +272,10 @@ Status TabletsChannel::reduce_mem_usage(int64_t mem_limit)
{
}
for (int i = 0; i < counter; i++) {
+ if (_broken_tablets.find(writers[i]->tablet_id()) !=
_broken_tablets.end()) {
+ // skip broken tablets
+ continue;
+ }
OLAPStatus st = writers[i]->wait_flush();
if (st != OLAP_SUCCESS) {
return Status::InternalError(fmt::format("failed to reduce mem
consumption by flushing memtable. err: {}", st));
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 9150a16343..96ef574261 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -458,12 +458,20 @@ int main(int argc, char** argv) {
#endif
#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) &&
!defined(THREAD_SANITIZER)
- doris::MemInfo::refresh_current_mem();
+ doris::MemInfo::refresh_allocator_mem();
#endif
doris::PerfCounters::refresh_proc_status();
+ int64_t allocator_cache_mem_diff =
+ doris::MemInfo::allocator_cache_mem() -
+
doris::ExecEnv::GetInstance()->allocator_cache_mem_tracker()->consumption();
+ doris::ExecEnv::GetInstance()->allocator_cache_mem_tracker()->consume(
+ allocator_cache_mem_diff);
+ CONSUME_THREAD_MEM_TRACKER(allocator_cache_mem_diff);
// 1s clear the expired task mem tracker, a query mem tracker is about
57 bytes.
doris::ExecEnv::GetInstance()->task_pool_mem_tracker_registry()->logout_task_mem_tracker();
+ // The process tracker print log usage interval is 1s to avoid a large
number of tasks being
+ // canceled when the process exceeds the mem limit, resulting in too
many duplicate logs.
doris::ExecEnv::GetInstance()->process_mem_tracker_raw()->enable_print_log_usage();
sleep(1);
}
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 3a8c6f494b..fe4e4e8f10 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -37,7 +37,15 @@ namespace doris {
bool MemInfo::_s_initialized = false;
int64_t MemInfo::_s_physical_mem = -1;
int64_t MemInfo::_s_mem_limit = -1;
-size_t MemInfo::_s_current_mem = 0;
+std::string MemInfo::_s_mem_limit_str = "";
+int64_t MemInfo::_s_hard_mem_limit = -1;
+size_t MemInfo::_s_allocator_physical_mem = 0;
+size_t MemInfo::_s_tcmalloc_pageheap_free_bytes = 0;
+size_t MemInfo::_s_tcmalloc_central_bytes = 0;
+size_t MemInfo::_s_tcmalloc_transfer_bytes = 0;
+size_t MemInfo::_s_tcmalloc_thread_bytes = 0;
+size_t MemInfo::_s_allocator_cache_mem = 0;
+std::string MemInfo::_s_allocator_cache_mem_str = "";
void MemInfo::init() {
// Read from /proc/meminfo
@@ -85,6 +93,8 @@ void MemInfo::init() {
bool is_percent = true;
_s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1,
_s_physical_mem, &is_percent);
+ _s_mem_limit_str = PrettyPrinter::print(_s_mem_limit, TUnit::BYTES);
+ _s_hard_mem_limit = _s_physical_mem - std::min(209715200L, _s_physical_mem
/ 10); // 200M
LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem,
TUnit::BYTES);
_s_initialized = true;
@@ -97,7 +107,8 @@ std::string MemInfo::debug_string() {
stream << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem,
TUnit::BYTES)
<< std::endl;
stream << "Memory Limt: " << PrettyPrinter::print(_s_mem_limit,
TUnit::BYTES) << std::endl;
- stream << "Current Usage: " << PrettyPrinter::print(_s_current_mem,
TUnit::BYTES) << std::endl;
+ stream << "Current Usage: " <<
PrettyPrinter::print(_s_allocator_physical_mem, TUnit::BYTES)
+ << std::endl;
stream << "CGroup Info: " << util.debug_string() << std::endl;
return stream.str();
}
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 53972ee925..d07f608f4e 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -23,12 +23,13 @@
#include <string>
#include "common/logging.h"
+#include "util/pretty_printer.h"
namespace doris {
// Provides the amount of physical memory available.
// Populated from /proc/meminfo.
-// TODO: Combine mem-info, cpu-info and disk-info into hardware-info?
+// TODO: Combine mem-info, cpu-info and disk-info into
hardware-info/perf_counters ?
class MemInfo {
public:
// Initialize MemInfo.
@@ -40,19 +41,37 @@ public:
return _s_physical_mem;
}
- static inline size_t current_mem() { return _s_current_mem; }
+ static inline size_t current_mem() { return _s_allocator_physical_mem; }
+ static inline size_t allocator_cache_mem() { return
_s_allocator_cache_mem; }
+ static inline std::string allocator_cache_mem_str() { return
_s_allocator_cache_mem_str; }
// Tcmalloc property `generic.total_physical_bytes` records the total
length of the virtual memory
// obtained by the process malloc, not the physical memory actually used
by the process in the OS.
- static inline void refresh_current_mem() {
+ static inline void refresh_allocator_mem() {
MallocExtension::instance()->GetNumericProperty("generic.total_physical_bytes",
- &_s_current_mem);
+
&_s_allocator_physical_mem);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes",
+
&_s_tcmalloc_pageheap_free_bytes);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.central_cache_free_bytes",
+
&_s_tcmalloc_central_bytes);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.transfer_cache_free_bytes",
+
&_s_tcmalloc_transfer_bytes);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.thread_cache_free_bytes",
+
&_s_tcmalloc_thread_bytes);
+ _s_allocator_cache_mem = _s_tcmalloc_pageheap_free_bytes +
_s_tcmalloc_central_bytes +
+ _s_tcmalloc_transfer_bytes +
_s_tcmalloc_thread_bytes;
+ _s_allocator_cache_mem_str =
PrettyPrinter::print(_s_allocator_cache_mem, TUnit::BYTES);
}
static inline int64_t mem_limit() {
DCHECK(_s_initialized);
return _s_mem_limit;
}
+ static inline std::string mem_limit_str() {
+ DCHECK(_s_initialized);
+ return _s_mem_limit_str;
+ }
+ static inline int64_t hard_mem_limit() { return _s_hard_mem_limit; }
static std::string debug_string();
@@ -60,7 +79,15 @@ private:
static bool _s_initialized;
static int64_t _s_physical_mem;
static int64_t _s_mem_limit;
- static size_t _s_current_mem;
+ static std::string _s_mem_limit_str;
+ static int64_t _s_hard_mem_limit;
+ static size_t _s_allocator_physical_mem;
+ static size_t _s_tcmalloc_pageheap_free_bytes;
+ static size_t _s_tcmalloc_central_bytes;
+ static size_t _s_tcmalloc_transfer_bytes;
+ static size_t _s_tcmalloc_thread_bytes;
+ static size_t _s_allocator_cache_mem;
+ static std::string _s_allocator_cache_mem_str;
};
} // namespace doris
diff --git a/be/src/util/perf_counters.cpp b/be/src/util/perf_counters.cpp
index fc62255f9d..d82c6c4afd 100644
--- a/be/src/util/perf_counters.cpp
+++ b/be/src/util/perf_counters.cpp
@@ -43,6 +43,9 @@ namespace doris {
static std::unordered_map<std::string, std::string> _process_state;
+int64_t PerfCounters::_vm_rss = 0;
+std::string PerfCounters::_vm_rss_str = "";
+
// This is the order of the counters in /proc/self/io
enum PERF_IO_IDX {
PROC_IO_READ = 0,
@@ -572,6 +575,9 @@ void PerfCounters::refresh_proc_status() {
}
if (statusinfo.is_open()) statusinfo.close();
+
+ _vm_rss = parse_bytes("status/VmRSS");
+ _vm_rss_str = PrettyPrinter::print(_vm_rss, TUnit::BYTES);
}
void PerfCounters::get_proc_status(ProcStatus* out) {
diff --git a/be/src/util/perf_counters.h b/be/src/util/perf_counters.h
index ca130e511c..89ff4c64d3 100644
--- a/be/src/util/perf_counters.h
+++ b/be/src/util/perf_counters.h
@@ -113,7 +113,8 @@ public:
static void refresh_proc_status();
static void get_proc_status(ProcStatus* out);
// Return the process actual physical memory in bytes.
- static inline int64_t get_vm_rss() { return parse_bytes("status/VmRSS"); }
+ static inline int64_t get_vm_rss() { return _vm_rss; }
+ static inline std::string get_vm_rss_str() { return _vm_rss_str; }
private:
// Copy constructor and assignment not allowed
@@ -157,6 +158,9 @@ private:
// System perf counters can be grouped together. The OS will update all
grouped counters
// at the same time. This is useful to better correlate counter values.
int _group_fd;
+
+ static int64_t _vm_rss;
+ static std::string _vm_rss_str;
};
} // namespace doris
diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp
index 1cdb50a563..df24281952 100644
--- a/be/src/util/system_metrics.cpp
+++ b/be/src/util/system_metrics.cpp
@@ -25,6 +25,7 @@
#include "gutil/strtoint.h" // for atoi64
#include "util/doris_metrics.h"
#include "util/mem_info.h"
+#include "util/perf_counters.h"
namespace doris {
@@ -313,7 +314,7 @@ void SystemMetrics::_install_memory_metrics(MetricEntity*
entity) {
}
void SystemMetrics::_update_memory_metrics() {
- _memory_metrics->memory_allocated_bytes->set_value(MemInfo::current_mem());
+
_memory_metrics->memory_allocated_bytes->set_value(PerfCounters::get_vm_rss());
}
void SystemMetrics::_install_disk_metrics(const std::set<std::string>&
disk_devices) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
