[
https://issues.apache.org/jira/browse/KUDU-3735?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marton Greber reassigned KUDU-3735:
-----------------------------------
Assignee: Marton Greber
> Data race in stack trace collection in debug-util.cc
> -----------------------------------------------------
>
> Key: KUDU-3735
> URL: https://issues.apache.org/jira/browse/KUDU-3735
> Project: Kudu
> Issue Type: Bug
> Components: util
> Reporter: Peter Rozsa
> Assignee: Marton Greber
> Priority: Minor
>
> ThreadSanitizer detects a data race in kudu::stack_trace_internal::SignalData
> during concurrent stack trace collection. The race occurs between atomic
> variable initialization in one thread and atomic operations on the same
> variable in another thread.
> The actual call is made from Impala, but it can occur in different
> environments too.
>
> {code:java}
> WARNING: ThreadSanitizer: data race (pid=747181) Atomic write of size 8 at
> 0x7b080011e0e8 by thread T54 (mutexes: write M1007816000152797856, write
> M1013445224811605968): #0 __tsan_atomic64_compare_exchange_val <null>
> (data-stream-test+0x24458a6) #1
> std::__atomic_base<long>::compare_exchange_strong(long&, long,
> std::memory_order, std::memory_order)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/atomic_base.h:509:9
> (data-stream-test+0x2bf1bb1) #2
> std::__atomic_base<long>::compare_exchange_strong(long&, long,
> std::memory_order)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/atomic_base.h:533
> (data-stream-test+0x2bf1bb1) #3 kudu::(anonymous
> namespace)::HandleStackTraceSignal(int, siginfo_t*, void*)
> repos/Impala/be/src/kudu/util/debug-util.cc:277 (data-stream-test+0x2bf1bb1)
> #4 __tsan::ProcessPendingSignals(__tsan::ThreadState*) <null>
> (data-stream-test+0x23f2810) #5 __gthread_mutex_lock(pthread_mutex_t*)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749:12
> (data-stream-test+0x2483276) #6 std::mutex::lock()
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_mutex.h:100:17
> (data-stream-test+0x248bda9) #7
> std::lock_guard<std::mutex>::lock_guard(std::mutex&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_mutex.h:159:19
> (data-stream-test+0x248b488) #8
> std::_V2::condition_variable_any::notify_one()
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/condition_variable:303:25
> (data-stream-test+0x2ee7766) #9
> impala::KrpcDataStreamSender::Channel::MarkDone(impala::Status const&)
> repos/Impala/be/src/runtime/krpc-data-stream-sender.cc:406:16
> (data-stream-test+0x2eeed88) #10
> impala::KrpcDataStreamSender::Channel::TransmitDataCompleteCb()
> repos/Impala/be/src/runtime/krpc-data-stream-sender.cc:543:5
> (data-stream-test+0x2eeff17) #11 boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>::operator()(impala::KrpcDataStreamSender::Channel*)
> const /boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:49:29
> (data-stream-test+0x2f056a6) #12 void
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*>
> >::operator()<boost::_mfi::mf0<void, impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>&, boost::_bi::list0&, int)
> /boost-1.74.0-p1/include/boost/bind/bind.hpp:259:9
> (data-stream-test+0x2f055fa) #13 boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> >::operator()() /boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
> (data-stream-test+0x2f05583) #14 void std::__invoke_impl<void,
> boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> >&>(std::__invoke_other, boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> >&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:60:14
> (data-stream-test+0x2f05501) #15 std::enable_if<is_invocable_r_v<void,
> boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> >&>, void>::type std::__invoke_r<void, boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> >&>(boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> >&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:110:2
> (data-stream-test+0x2f05481) #16 std::_Function_handler<void (),
> boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> impala::KrpcDataStreamSender::Channel>,
> boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
> > >::_M_invoke(std::_Any_data const&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:291:9
> (data-stream-test+0x2f052f1) #17 std::function<void ()>::operator()()
> const
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:622:14
> (data-stream-test+0x2c3eedc) #18 kudu::rpc::OutboundCall::CallCallback()
> repos/Impala/be/src/kudu/rpc/outbound_call.cc:309:5
> (data-stream-test+0x2d1b644) #19
> kudu::rpc::OutboundCall::SetResponse(std::unique_ptr<kudu::rpc::CallResponse,
> std::default_delete<kudu::rpc::CallResponse> >)
> repos/Impala/be/src/kudu/rpc/outbound_call.cc:341:5
> (data-stream-test+0x2d1b85e) #20
> kudu::rpc::Connection::HandleCallResponse(std::unique_ptr<kudu::rpc::InboundTransfer,
> std::default_delete<kudu::rpc::InboundTransfer> >)
> repos/Impala/be/src/kudu/rpc/connection.cc:744:14
> (data-stream-test+0x2cfa1de) #21
> kudu::rpc::Connection::ReadHandler(ev::io&, int)
> repos/Impala/be/src/kudu/rpc/connection.cc:682:7 (data-stream-test+0x2cf9a2f)
> #22 void ev::base<ev_io, ev::io>::method_thunk<kudu::rpc::Connection,
> &kudu::rpc::Connection::ReadHandler>(ev_loop*, ev_io*, int)
> /libev-4.20-p1/include/ev++.h:479:7 (data-stream-test+0x2d04e1a) #23
> ev_invoke_pending /mnt/source/libev/libev-4.20-p1/ev.c:3155:11
> (data-stream-test+0x571c222) #24 ev_run
> /mnt/source/libev/libev-4.20-p1/ev.c:3555:7 (data-stream-test+0x571f574)
> #25 ev_run /mnt/source/libev/libev-4.20-p1/ev.c:3402:1
> (data-stream-test+0x571f574) #26 kudu::rpc::ReactorThread::RunThread()
> repos/Impala/be/src/kudu/rpc/reactor.cc:503:9 (data-stream-test+0x2d232b4)
> #27 kudu::rpc::ReactorThread::Init()::$_0::operator()() const
> repos/Impala/be/src/kudu/rpc/reactor.cc:194:48 (data-stream-test+0x2d253c1)
> #28 void std::__invoke_impl<void,
> kudu::rpc::ReactorThread::Init()::$_0&>(std::__invoke_other,
> kudu::rpc::ReactorThread::Init()::$_0&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:60:14
> (data-stream-test+0x2d25351) #29 std::enable_if<is_invocable_r_v<void,
> kudu::rpc::ReactorThread::Init()::$_0&>, void>::type std::__invoke_r<void,
> kudu::rpc::ReactorThread::Init()::$_0&>(kudu::rpc::ReactorThread::Init()::$_0&)
>
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:110:2
> (data-stream-test+0x2d252d1) #30 std::_Function_handler<void (),
> kudu::rpc::ReactorThread::Init()::$_0>::_M_invoke(std::_Any_data const&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:291:9
> (data-stream-test+0x2d251a1) #31 std::function<void ()>::operator()()
> const
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:622:14
> (data-stream-test+0x2c3eedc) #32 kudu::Thread::SuperviseThread(void*)
> repos/Impala/be/src/kudu/util/thread.cc:691:3 (data-stream-test+0x2cb0e56)
> Previous write of size 8 at 0x7b080011e0e8 by thread T44 (mutexes: write
> M936321304528812448): #0 std::__atomic_base<long>::__atomic_base(long)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/atomic_base.h:286:58
> (data-stream-test+0x2bf415d) #1 std::atomic<long>::atomic(long)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/atomic:816:56
> (data-stream-test+0x2bf4100) #2
> kudu::stack_trace_internal::SignalData::SignalData()
> repos/Impala/be/src/kudu/util/debug-util.cc:243:38
> (data-stream-test+0x2bf331c) #3
> kudu::StackTraceCollector::TriggerAsync(long, kudu::StackTrace*)
> repos/Impala/be/src/kudu/util/debug-util.cc:436:40
> (data-stream-test+0x2bf0151) #4 kudu::GetThreadStack(long,
> kudu::StackTrace*) repos/Impala/be/src/kudu/util/debug-util.cc:507:3
> (data-stream-test+0x2bf05fe) #5 kudu::DumpThreadStack[abi:cxx11](long)
> repos/Impala/be/src/kudu/util/debug-util.cc:514:14
> (data-stream-test+0x2bf071c) #6 kudu::KernelStackWatchdog::RunThread()
> repos/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:185:31
> (data-stream-test+0x2cc5e3e) #7
> kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0::operator()() const
> repos/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:71:3
> (data-stream-test+0x2cc6631) #8 void std::__invoke_impl<void,
> kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&>(std::__invoke_other,
> kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:60:14
> (data-stream-test+0x2cc65c1) #9 std::enable_if<is_invocable_r_v<void,
> kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&>, void>::type
> std::__invoke_r<void,
> kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&>(kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&)
>
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:110:2
> (data-stream-test+0x2cc6541) #10 std::_Function_handler<void (),
> kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0>::_M_invoke(std::_Any_data
> const&)
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:291:9
> (data-stream-test+0x2cc6411) #11 std::function<void ()>::operator()()
> const
> /gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:622:14
> (data-stream-test+0x2c3eedc) #12 kudu::Thread::SuperviseThread(void*)
> repos/Impala/be/src/kudu/util/thread.cc:691:3
> (data-stream-test+0x2cb0e56){code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
