[ https://issues.apache.org/jira/browse/IMPALA-9739?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Bikramjeet Vig resolved IMPALA-9739. ------------------------------------ Fix Version/s: Impala 4.0 Resolution: Fixed > TSAN data races during impalad shutdown > --------------------------------------- > > Key: IMPALA-9739 > URL: https://issues.apache.org/jira/browse/IMPALA-9739 > Project: IMPALA > Issue Type: Sub-task > Components: Backend > Reporter: Sahil Takiar > Assignee: Bikramjeet Vig > Priority: Major > Fix For: Impala 4.0 > > > A TSAN run of the custom cluster tests shows several instances of the > following data race during impalad shutdown: > {code:java} > WARNING: ThreadSanitizer: data race (pid=12660) > Read of size 8 at 0x000007786f60 by thread T338: > #0 std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >::~unique_ptr() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/unique_ptr.h:235:6 > (impalad+0x19bd895) > #1 at_exit_wrapper(void*) > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:361 > (impalad+0x191cf13) > #2 impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2::operator()() const > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2622:57 > (impalad+0x21bd871) > #3 > boost::detail::function::void_function_obj_invoker0<impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2, > void>::invoke(boost::detail::function::function_buffer&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 > (impalad+0x21bd6d9) > #4 boost::function0<void>::operator()() const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 > (impalad+0x1e192b1) > #5 impala::Thread::SuperviseThread(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:360:3 > (impalad+0x23df196) > #6 void boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 > (impalad+0x23e735c) > #7 boost::_bi::bind_t<void, void (*)(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x23e7273) > #8 boost::detail::thread_data<boost::_bi::bind_t<void, void > (*)(std::string const&, std::string const&, boost::function<void ()>, > impala::ThreadDebugInfo const*, impala::Promise<long, > (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > > >::run() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 > (impalad+0x23e6f60) > #9 thread_proxy <null> (impalad+0x30e44f9) > Previous write of size 8 at 0x000007786f60 by main thread: > #0 void std::swap<impala::Thread*>(impala::Thread*&, impala::Thread*&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/move.h:176:11 > (impalad+0x221f370) > #1 std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >::reset(impala::Thread*) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/unique_ptr.h:342:2 > (impalad+0x221a5ab) > #2 std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >::operator=(std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >&&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/unique_ptr.h:251:2 > (impalad+0x23e1ad8) > #3 impala::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:329:11 > (impalad+0x23decac) > #4 impala::Status impala::Thread::Create<void (*)()>(std::string const&, > std::string const&, void (* const&)(), std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.h:74:12 > (impalad+0x1a1cd8c) > #5 impala::StartImpalaShutdownSignalHandlerThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/common/init.cc:407:10 > (impalad+0x1a1c1e8) > #6 ImpaladMain(int, char**) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impalad-main.cc:96:3 > (impalad+0x21a525a) > #7 main > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/daemon-main.cc:37:12 > (impalad+0x19b860a) > As if synchronized via sleep: > #0 nanosleep > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:343 > (impalad+0x19523ea) > #1 void std::this_thread::sleep_for<long, std::ratio<1l, 1000l> > >(std::chrono::duration<long, std::ratio<1l, 1000l> > const&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/thread:279:2 > (impalad+0x23e9102) > #2 impala::SleepForMs(long) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/time.cc:31:3 > (impalad+0x23e853d) > #3 impala::ImpalaServer::ShutdownThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2644:5 > (impalad+0x21bc6e9) > #4 impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2::operator()() const > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2622:57 > (impalad+0x21bd871) > #5 > boost::detail::function::void_function_obj_invoker0<impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2, > void>::invoke(boost::detail::function::function_buffer&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 > (impalad+0x21bd6d9) > #6 boost::function0<void>::operator()() const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 > (impalad+0x1e192b1) > #7 impala::Thread::SuperviseThread(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:360:3 > (impalad+0x23df196) > #8 void boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 > (impalad+0x23e735c) > #9 boost::_bi::bind_t<void, void (*)(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x23e7273) > #10 boost::detail::thread_data<boost::_bi::bind_t<void, void > (*)(std::string const&, std::string const&, boost::function<void ()>, > impala::ThreadDebugInfo const*, impala::Promise<long, > (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > > >::run() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 > (impalad+0x23e6f60) > #11 thread_proxy <null> (impalad+0x30e44f9) > Location is global 'shutdown_signal_handler_thread' of size 8 at > 0x000007786f60 (impalad+0x000007786f60) > Thread T338 (tid=14503, running) created by thread T100 at: > #0 pthread_create > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889 > (impalad+0x194d8bb) > #1 boost::thread::start_thread_noexcept() <null> (impalad+0x30e3889) > #2 boost::thread::thread<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, > boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, > (impala::PromiseMode)0>*>(void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, > boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, > (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13 > (impalad+0x23e196b) > #3 impala::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:317:13 > (impalad+0x23deb35) > #4 impala::Status > impala::Thread::Create<impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2>(std::string const&, std::string const&, > impala::ImpalaServer::StartShutdown(long, impala::ShutdownStatusPB*)::$_2 > const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.h:74:12 > (impalad+0x21bc676) > #5 impala::ImpalaServer::StartShutdown(long, impala::ShutdownStatusPB*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2622:9 > (impalad+0x21bc415) > #6 impala::ControlService::RemoteShutdown(impala::RemoteShutdownParamsPB > const*, impala::RemoteShutdownResultPB*, kudu::rpc::RpcContext*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/control-service.cc:239:60 > (impalad+0x2179844) > #7 > impala::ControlServiceIf::ControlServiceIf(scoped_refptr<kudu::MetricEntity> > const&, scoped_refptr<kudu::rpc::ResultTracker> > const&)::$_7::operator()(google::protobuf::Message const*, > google::protobuf::Message*, kudu::rpc::RpcContext*) const > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/generated-sources/gen-cpp/control_service.service.cc:130:13 > (impalad+0x2227864) > #8 std::_Function_handler<void (google::protobuf::Message const*, > google::protobuf::Message*, kudu::rpc::RpcContext*), > impala::ControlServiceIf::ControlServiceIf(scoped_refptr<kudu::MetricEntity> > const&, scoped_refptr<kudu::rpc::ResultTracker> > const&)::$_7>::_M_invoke(std::_Any_data const&, google::protobuf::Message > const*, google::protobuf::Message*, kudu::rpc::RpcContext*) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/functional:2039:2 > (impalad+0x222764e) > #9 std::function<void (google::protobuf::Message const*, > google::protobuf::Message*, > kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, > google::protobuf::Message*, kudu::rpc::RpcContext*) const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/functional:2439:14 > (impalad+0x28ce678) > #10 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/kudu/rpc/service_if.cc:139:3 > (impalad+0x28cdfde) > #11 impala::ImpalaServicePool::RunThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/rpc/impala-service-pool.cc:272:15 > (impalad+0x1f3af6f) > #12 boost::_mfi::mf0<void, > impala::ImpalaServicePool>::operator()(impala::ImpalaServicePool*) const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:49:29 > (impalad+0x1f414d6) > #13 void boost::_bi::list1<boost::_bi::value<impala::ImpalaServicePool*> > >::operator()<boost::_mfi::mf0<void, impala::ImpalaServicePool>, > boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, > impala::ImpalaServicePool>&, boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:259:9 > (impalad+0x1f4142a) > #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, > impala::ImpalaServicePool>, > boost::_bi::list1<boost::_bi::value<impala::ImpalaServicePool*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x1f413b3) > #15 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf0<void, impala::ImpalaServicePool>, > boost::_bi::list1<boost::_bi::value<impala::ImpalaServicePool*> > >, > void>::invoke(boost::detail::function::function_buffer&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 > (impalad+0x1f411a9) > #16 boost::function0<void>::operator()() const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 > (impalad+0x1e192b1) > #17 impala::Thread::SuperviseThread(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:360:3 > (impalad+0x23df196) > #18 void boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 > (impalad+0x23e735c) > #19 boost::_bi::bind_t<void, void (*)(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x23e7273) > #20 boost::detail::thread_data<boost::_bi::bind_t<void, void > (*)(std::string const&, std::string const&, boost::function<void ()>, > impala::ThreadDebugInfo const*, impala::Promise<long, > (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > > >::run() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 > (impalad+0x23e6f60) > #21 thread_proxy <null> (impalad+0x30e44f9) > SUMMARY: ThreadSanitizer: data race > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/unique_ptr.h:235:6 > in std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >::~unique_ptr() > {code} > There is another (seemingly related) data race as well; the main difference > is that the race occurs inside the scoped_ptr rather than a unique_ptr: > {code:java} > WARNING: ThreadSanitizer: data race (pid=12670) > Read of size 8 at 0x7b08002c5560 by thread T336: > #0 boost::scoped_ptr<boost::thread>::~scoped_ptr() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/smart_ptr/scoped_ptr.hpp:88:32 > (impalad+0x19be459) > #1 impala::Thread::~Thread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.h:50:7 > (impalad+0x19be42b) > #2 std::default_delete<impala::Thread>::operator()(impala::Thread*) const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/unique_ptr.h:76:2 > (impalad+0x19be28e) > #3 std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >::~unique_ptr() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/unique_ptr.h:236:4 > (impalad+0x19bd8b9) > #4 at_exit_wrapper(void*) > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:361 > (impalad+0x191cf13) > #5 impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2::operator()() const > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2622:57 > (impalad+0x21bd871) > #6 > boost::detail::function::void_function_obj_invoker0<impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2, > void>::invoke(boost::detail::function::function_buffer&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 > (impalad+0x21bd6d9) > #7 boost::function0<void>::operator()() const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 > (impalad+0x1e192b1) > #8 impala::Thread::SuperviseThread(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:360:3 > (impalad+0x23df196) > #9 void boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 > (impalad+0x23e735c) > #10 boost::_bi::bind_t<void, void (*)(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x23e7273) > #11 boost::detail::thread_data<boost::_bi::bind_t<void, void > (*)(std::string const&, std::string const&, boost::function<void ()>, > impala::ThreadDebugInfo const*, impala::Promise<long, > (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > > >::run() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 > (impalad+0x23e6f60) > #12 thread_proxy <null> (impalad+0x30e44f9) > Previous write of size 8 at 0x7b08002c5560 by main thread: > #0 > boost::scoped_ptr<boost::thread>::swap(boost::scoped_ptr<boost::thread>&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/smart_ptr/scoped_ptr.hpp:120:14 > (impalad+0x239089f) > #1 boost::scoped_ptr<boost::thread>::reset(boost::thread*) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/smart_ptr/scoped_ptr.hpp:94:22 > (impalad+0x238ca14) > #2 impala::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:316:16 > (impalad+0x23deb43) > #3 impala::Status impala::Thread::Create<void (*)()>(std::string const&, > std::string const&, void (* const&)(), std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.h:74:12 > (impalad+0x1a1cd8c) > #4 impala::StartImpalaShutdownSignalHandlerThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/common/init.cc:407:10 > (impalad+0x1a1c1e8) > #5 ImpaladMain(int, char**) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impalad-main.cc:96:3 > (impalad+0x21a525a) > #6 main > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/daemon-main.cc:37:12 > (impalad+0x19b860a) > As if synchronized via sleep: > #0 nanosleep > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:343 > (impalad+0x19523ea) > #1 void std::this_thread::sleep_for<long, std::ratio<1l, 1000l> > >(std::chrono::duration<long, std::ratio<1l, 1000l> > const&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/thread:279:2 > (impalad+0x23e9102) > #2 impala::SleepForMs(long) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/time.cc:31:3 > (impalad+0x23e853d) > #3 impala::ImpalaServer::ShutdownThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2644:5 > (impalad+0x21bc6e9) > #4 impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2::operator()() const > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2622:57 > (impalad+0x21bd871) > #5 > boost::detail::function::void_function_obj_invoker0<impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2, > void>::invoke(boost::detail::function::function_buffer&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 > (impalad+0x21bd6d9) > #6 boost::function0<void>::operator()() const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 > (impalad+0x1e192b1) > #7 impala::Thread::SuperviseThread(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:360:3 > (impalad+0x23df196) > #8 void boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 > (impalad+0x23e735c) > #9 boost::_bi::bind_t<void, void (*)(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x23e7273) > #10 boost::detail::thread_data<boost::_bi::bind_t<void, void > (*)(std::string const&, std::string const&, boost::function<void ()>, > impala::ThreadDebugInfo const*, impala::Promise<long, > (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > > >::run() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 > (impalad+0x23e6f60) > #11 thread_proxy <null> (impalad+0x30e44f9) > Location is heap block of size 32 at 0x7b08002c5560 allocated by main > thread: > #0 operator new(unsigned long) > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:44 > (impalad+0x19b5ed3) > #1 impala::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:313:24 > (impalad+0x23dea7f) > #2 impala::Status impala::Thread::Create<void (*)()>(std::string const&, > std::string const&, void (* const&)(), std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.h:74:12 > (impalad+0x1a1cd8c) > #3 impala::StartImpalaShutdownSignalHandlerThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/common/init.cc:407:10 > (impalad+0x1a1c1e8) > #4 ImpaladMain(int, char**) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impalad-main.cc:96:3 > (impalad+0x21a525a) > #5 main > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/daemon-main.cc:37:12 > (impalad+0x19b860a) > Thread T336 (tid=14509, running) created by thread T100 at: > #0 pthread_create > /mnt/source/llvm/llvm-5.0.1.src-p2/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889 > (impalad+0x194d8bb) > #1 boost::thread::start_thread_noexcept() <null> (impalad+0x30e3889) > #2 boost::thread::thread<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, > boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, > (impala::PromiseMode)0>*>(void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, > boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, > (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13 > (impalad+0x23e196b) > #3 impala::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:317:13 > (impalad+0x23deb35) > #4 impala::Status > impala::Thread::Create<impala::ImpalaServer::StartShutdown(long, > impala::ShutdownStatusPB*)::$_2>(std::string const&, std::string const&, > impala::ImpalaServer::StartShutdown(long, impala::ShutdownStatusPB*)::$_2 > const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> > >*, bool) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.h:74:12 > (impalad+0x21bc676) > #5 impala::ImpalaServer::StartShutdown(long, impala::ShutdownStatusPB*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/impala-server.cc:2622:9 > (impalad+0x21bc415) > #6 impala::ControlService::RemoteShutdown(impala::RemoteShutdownParamsPB > const*, impala::RemoteShutdownResultPB*, kudu::rpc::RpcContext*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/service/control-service.cc:239:60 > (impalad+0x2179844) > #7 > impala::ControlServiceIf::ControlServiceIf(scoped_refptr<kudu::MetricEntity> > const&, scoped_refptr<kudu::rpc::ResultTracker> > const&)::$_7::operator()(google::protobuf::Message const*, > google::protobuf::Message*, kudu::rpc::RpcContext*) const > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/generated-sources/gen-cpp/control_service.service.cc:130:13 > (impalad+0x2227864) > #8 std::_Function_handler<void (google::protobuf::Message const*, > google::protobuf::Message*, kudu::rpc::RpcContext*), > impala::ControlServiceIf::ControlServiceIf(scoped_refptr<kudu::MetricEntity> > const&, scoped_refptr<kudu::rpc::ResultTracker> > const&)::$_7>::_M_invoke(std::_Any_data const&, google::protobuf::Message > const*, google::protobuf::Message*, kudu::rpc::RpcContext*) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/functional:2039:2 > (impalad+0x222764e) > #9 std::function<void (google::protobuf::Message const*, > google::protobuf::Message*, > kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, > google::protobuf::Message*, kudu::rpc::RpcContext*) const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/functional:2439:14 > (impalad+0x28ce678) > #10 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/kudu/rpc/service_if.cc:139:3 > (impalad+0x28cdfde) > #11 impala::ImpalaServicePool::RunThread() > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/rpc/impala-service-pool.cc:272:15 > (impalad+0x1f3af6f) > #12 boost::_mfi::mf0<void, > impala::ImpalaServicePool>::operator()(impala::ImpalaServicePool*) const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:49:29 > (impalad+0x1f414d6) > #13 void boost::_bi::list1<boost::_bi::value<impala::ImpalaServicePool*> > >::operator()<boost::_mfi::mf0<void, impala::ImpalaServicePool>, > boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, > impala::ImpalaServicePool>&, boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:259:9 > (impalad+0x1f4142a) > #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, > impala::ImpalaServicePool>, > boost::_bi::list1<boost::_bi::value<impala::ImpalaServicePool*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x1f413b3) > #15 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf0<void, impala::ImpalaServicePool>, > boost::_bi::list1<boost::_bi::value<impala::ImpalaServicePool*> > >, > void>::invoke(boost::detail::function::function_buffer&) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 > (impalad+0x1f411a9) > #16 boost::function0<void>::operator()() const > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 > (impalad+0x1e192b1) > #17 impala::Thread::SuperviseThread(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*) > /data/jenkins/workspace/impala-private-parameterized/repos/Impala/be/src/util/thread.cc:360:3 > (impalad+0x23df196) > #18 void boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 > (impalad+0x23e735c) > #19 boost::_bi::bind_t<void, void (*)(std::string const&, std::string > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), > boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::operator()() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 > (impalad+0x23e7273) > #20 boost::detail::thread_data<boost::_bi::bind_t<void, void > (*)(std::string const&, std::string const&, boost::function<void ()>, > impala::ThreadDebugInfo const*, impala::Promise<long, > (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, > boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, > boost::_bi::value<impala::ThreadDebugInfo*>, > boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > > >::run() > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 > (impalad+0x23e6f60) > #21 thread_proxy <null> (impalad+0x30e44f9) > SUMMARY: ThreadSanitizer: data race > /data/jenkins/workspace/impala-private-parameterized/Impala-Toolchain/boost-1.61.0-p2/include/boost/smart_ptr/scoped_ptr.hpp:88:32 > in boost::scoped_ptr<boost::thread>::~scoped_ptr() > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-all-unsubscr...@impala.apache.org For additional commands, e-mail: issues-all-h...@impala.apache.org