This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.1 by this push:
     new 8aa2fafff4b [bugfix](asan core) should use weak ptr in rpc context to 
release rf context during query context deconstructed (#46565)
8aa2fafff4b is described below

commit 8aa2fafff4b35db68ab6e03754d5c510545f174a
Author: yiguolei <[email protected]>
AuthorDate: Wed Jan 8 09:30:29 2025 +0800

    [bugfix](asan core) should use weak ptr in rpc context to release rf 
context during query context deconstructed (#46565)
    
    cherry-pick  #38653
    
    Issue Number: close #xxx
    F20240731 10:38:53.012670 20850 mem_tracker_limiter.cpp:135] mem tracker
    label: Query#Id=7539c7d0087442b7-a1cda6392053548a, consumption: 52, peak
    consumption: 1155332, mem tracker not equal to 0 when mem tracker
    destruct, this usually means that memory tracking is inaccurate and
    SCOPED_ATTACH_TASK and SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER are not
    used correctly. 1. For query and load, memory leaks may have occurred,
    it is expected that the query mem tracker will be bound to the thread
    context using SCOPED_ATTACH_TASK and
    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER before all memory alloc and
    free. 2. If a memory alloc is recorded by this tracker, it is expected
    that be recorded in this tracker when memory is freed. 3. Merge the
    remaining memory tracking value by this tracker into Orphan, if you
    observe that Orphan is not equal to 0 in the mem tracker web or log,
    this indicates that there may be a memory leak. 4. If you need to
    transfer memory tracking value between two trackers, can use
    transfer_to..[Address Sanitizer]:
    0x606002867d80, size 52, strack trace:
    0# Allocator::alloc_impl(unsigned long, unsigned long) 1# void*
    phmap::priv::Allocate<4ul, doris::vectorized::Allocator_
    >(doris::vectorized::Allocator_*, unsigned long)
    2# phmap::priv::raw_hash_set, phmap::Hash, phmap::EqualTo,
    doris::vectorized::Allocator_ >::initialize_slots(unsigned long) 3#
    phmap::priv::raw_hash_set, phmap::Hash, phmap::EqualTo,
    doris::vectorized::Allocator_ >::resize(unsigned long) 4#
    phmap::priv::raw_hash_set, phmap::Hash, phmap::EqualTo,
    doris::vectorized::Allocator_ >::prepare_insert(unsigned long) 5#
    std::pair phmap::priv::raw_hash_set, phmap::Hash, phmap::EqualTo,
    doris::vectorized::Allocator_ >::find_or_prepare_insert(int const&,
    unsigned long)
    6# std::pair, phmap::Hash, phmap::EqualTo, doris::vectorized::Allocator_
    >::iterator, bool> phmap::priv::raw_hash_set, phmap::Hash,
    phmap::EqualTo, doris::vectorized::Allocator_
    >::emplace_decomposable(int const&, unsigned long, int const&)
    7# std::pair, phmap::Hash, phmap::EqualTo, doris::vectorized::Allocator_
    >::iterator, bool> phmap::priv::raw_hash_set, phmap::Hash,
    phmap::EqualTo, doris::vectorized::Allocator_
    >::EmplaceDecomposable::operator()(int const&, int const&) const
    8# doris::HybridSet<(doris::PrimitiveType)5, doris::DynamicContainer,
    doris::vectorized::ColumnVector >::insert(void const*) 9#
    doris::HybridSetBase::insert(doris::HybridSetBase*) 10#
    doris::RuntimePredicateWrapper::merge(doris::RuntimePredicateWrapper
    const*)
    11# doris::IRuntimeFilter::merge_from(doris::RuntimePredicateWrapper
    const*)
    12#
    
    doris::RuntimeFilterMergeControllerEntity::merge(doris::PMergeFilterRequest
    const*, butil::IOBufAsZeroCopyInputStream*)
    13# doris::FragmentMgr::merge_filter(doris::PMergeFilterRequest const*,
    butil::IOBufAsZeroCopyInputStream*)
    14# std::_Function_handler::_M_invoke(std::_Any_data const&) 15#
    doris::WorkThreadPool::work_thread(int)
    16# execute_native_thread_routine
    17# ?
    18# ?
---
 be/src/exprs/runtime_filter.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index aa0d69ae5f0..80d9494ac9e 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -1077,7 +1077,10 @@ Status IRuntimeFilter::publish(bool publish_local) {
 class SyncSizeClosure : public AutoReleaseClosure<PSendFilterSizeRequest,
                                                   
DummyBrpcCallback<PSendFilterSizeResponse>> {
     std::shared_ptr<pipeline::Dependency> _dependency;
-    RuntimeFilterContextSPtr _rf_context;
+    // Should use weak ptr here, because when query context deconstructs, 
should also delete runtime filter
+    // context, it not the memory is not released. And rpc is in another 
thread, it will hold rf context
+    // after query context because the rpc is not returned.
+    std::weak_ptr<RuntimeFilterContext> _rf_context;
     std::string _rf_debug_info;
 
     using Base =
@@ -1085,22 +1088,22 @@ class SyncSizeClosure : public 
AutoReleaseClosure<PSendFilterSizeRequest,
     ENABLE_FACTORY_CREATOR(SyncSizeClosure);
 
     void _process_if_rpc_failed() override {
-        Defer defer {[&]() { 
((pipeline::CountedFinishDependency*)_dependency.get())->sub(); }};
-        auto ctx = _rf_context;
+        auto ctx = _rf_context.lock();
         if (!ctx) {
             return;
         }
 
+        Defer defer {[&]() { 
((pipeline::CountedFinishDependency*)_dependency.get())->sub(); }};
         ctx->err_msg = cntl_->ErrorText();
         Base::_process_if_rpc_failed();
     }
 
     void _process_if_meet_error_status(const Status& status) override {
-        Defer defer {[&]() { 
((pipeline::CountedFinishDependency*)_dependency.get())->sub(); }};
-        auto ctx = _rf_context;
+        auto ctx = _rf_context.lock();
         if (!ctx) {
             return;
         }
+        Defer defer {[&]() { 
((pipeline::CountedFinishDependency*)_dependency.get())->sub(); }};
 
         if (status.is<ErrorCode::END_OF_FILE>()) {
             // rf merger backend may finished before rf's send_filter_size, we 
just ignore filter in this case.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to