[ https://issues.apache.org/jira/browse/IMPALA-9314?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17021295#comment-17021295 ]
Sahil Takiar commented on IMPALA-9314: -------------------------------------- Data race on {{stop_}} {code:java} WARNING: ThreadSanitizer: data race (pid=29040) Write of size 1 at 0x7b4400001718 by main thread: #0 apache::thrift::server::TAcceptQueueServer::stop() /home/systest/Impala/be/src/rpc/TAcceptQueueServer.h:73:11 (statestore-test+0x271a7ed) #1 impala::ThriftServer::StopForTesting() /home/systest/Impala/be/src/rpc/thrift-server.cc:466:12 (statestore-test+0x27068a1) #2 impala::Statestore::ShutdownForTesting() /home/systest/Impala/be/src/statestore/statestore.cc:1072:21 (statestore-test+0x1d7d5ed) #3 impala::SslSmokeTestHelper(std::string const&, std::string const&, bool) /home/systest/Impala/be/src/statestore/statestore-test.cc:98:15 (statestore-test+0x18fc1a9) #4 impala::StatestoreSslTest_ValidCertSmokeTest_Test::TestBody() /home/systest/Impala/be/src/statestore/statestore-test.cc:110:3 (statestore-test+0x18fc638) #5 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (statestore-test+0x4105212) #6 __libc_start_main /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f) Previous read of size 1 at 0x7b4400001718 by thread T6: #0 apache::thrift::server::TAcceptQueueServer::serve() /home/systest/Impala/be/src/rpc/TAcceptQueueServer.cpp:334:11 (statestore-test+0x2718573) #1 impala::ThriftServer::ThriftServerEventProcessor::Supervise() /home/systest/Impala/be/src/rpc/thrift-server.cc:152:30 (statestore-test+0x2704b86) #2 boost::_mfi::mf0<void, impala::ThriftServer::ThriftServerEventProcessor>::operator()(impala::ThriftServer::ThriftServerEventProcessor*) const /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:49:16 (statestore-test+0x270af36) #3 void boost::_bi::list1<boost::_bi::value<impala::ThriftServer::ThriftServerEventProcessor*> >::operator()<boost::_mfi::mf0<void, impala::ThriftServer::ThriftServerEventProcessor>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, impala::ThriftSer ver::ThriftServerEventProcessor>&, boost::_bi::list0&, int) /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:253:9 (statestore-test+0x270ae8a) #4 boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ThriftServer::ThriftServerEventProcessor>, boost::_bi::list1<boost::_bi::value<impala::ThriftServer::ThriftServerEventProcessor*> > >::operator()() /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boos t/bind/bind_template.hpp:20:16 (statestore-test+0x270ae13) #5 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ThriftServer::ThriftServerEventProcessor>, boost::_bi::list1<boost::_bi::value<impala::ThriftServer::ThriftServerEventProcessor*> > >, void>::invoke(boost: :detail::function::function_buffer&) /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153:11 (statestore-test+0x270ac09) #6 boost::function0<void>::operator()() const /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:766:14 (statestore-test+0x19420c1) #7 impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) /home/systest/Impala/be/src/util/thread.cc:360:3 (statestore-test+0x1ebd266) #8 void boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:525:9 (statestore-test+0x1ec58cc) #9 boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20:16 (stat estore-test+0x1ec57e3) #10 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::string>, boos t::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/deta il/thread.hpp:116:17 (statestore-test+0x1ec54d0) #11 thread_proxy <null> (statestore-test+0x2f317a9) Location is heap block of size 288 at 0x7b4400001680 allocated by main thread: #0 operator new(unsigned long) /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:44 (statestore-test+0x18f8a23) #1 impala::ThriftServer::Start() /home/systest/Impala/be/src/rpc/thrift-server.cc:436:17 (statestore-test+0x27062b2) #2 impala::Statestore::Init(int) /home/systest/Impala/be/src/statestore/statestore.cc:478:59 (statestore-test+0x1d7a431) #3 impala::SslSmokeTestHelper(std::string const&, std::string const&, bool) /home/systest/Impala/be/src/statestore/statestore-test.cc:90:45 (statestore-test+0x18fbe96) #4 impala::StatestoreSslTest_ValidCertSmokeTest_Test::TestBody() /home/systest/Impala/be/src/statestore/statestore-test.cc:110:3 (statestore-test+0x18fc638) #5 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (statestore-test+0x4105212) #6 __libc_start_main /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f) Location is heap block of size 288 at 0x7b4400001680 allocated by main thread: #0 operator new(unsigned long) /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:44 (statestore-test+0x18f8a23) #1 impala::ThriftServer::Start() /home/systest/Impala/be/src/rpc/thrift-server.cc:436:17 (statestore-test+0x27062b2) #2 impala::Statestore::Init(int) /home/systest/Impala/be/src/statestore/statestore.cc:478:59 (statestore-test+0x1d7a431) #3 impala::SslSmokeTestHelper(std::string const&, std::string const&, bool) /home/systest/Impala/be/src/statestore/statestore-test.cc:90:45 (statestore-test+0x18fbe96) #4 impala::StatestoreSslTest_ValidCertSmokeTest_Test::TestBody() /home/systest/Impala/be/src/statestore/statestore-test.cc:110:3 (statestore-test+0x18fc638) #5 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (statestore-test+0x4105212) #6 __libc_start_main /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f) Thread T6 (tid=29107, running) created by main thread at: #0 pthread_create /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889 (statestore-test+0x18903bb) #1 boost::thread::start_thread_noexcept() <null> (statestore-test+0x2f30b89) #2 boost::thread::thread<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, boost::function<void ()>, impala::ThreadDebugInfo*, impala: :Promise<long, (impala::PromiseMode)0>*>(void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, boost::function<void ()>, impala::ThreadDebugIn fo*, impala::Promise<long, (impala::PromiseMode)0>*) /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:419:13 (statestore-test+0x1ebf91b) #3 impala::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) /home/systest/Impala/be/src/util/thread.cc:317:13 (statestore-test+0x1ebcc05) #4 impala::Status impala::Thread::Create<void (impala::ThriftServer::ThriftServerEventProcessor::*)(), impala::ThriftServer::ThriftServerEventProcessor*>(std::string const&, std::string const&, void (impala::ThriftServer::ThriftServerEventProcessor::* const&)(), imp ala::ThriftServer::ThriftServerEventProcessor* const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) /home/systest/Impala/be/src/util/thread.h:81:12 (statestore-test+0x27074c7) #5 impala::ThriftServer::ThriftServerEventProcessor::StartAndWaitForServer() /home/systest/Impala/be/src/rpc/thrift-server.cc:116:43 (statestore-test+0x2704722) #6 impala::ThriftServer::Start() /home/systest/Impala/be/src/rpc/thrift-server.cc:447:60 (statestore-test+0x27063fc) #7 impala::Statestore::Init(int) /home/systest/Impala/be/src/statestore/statestore.cc:478:59 (statestore-test+0x1d7a431) #8 impala::SslSmokeTestHelper(std::string const&, std::string const&, bool) /home/systest/Impala/be/src/statestore/statestore-test.cc:90:45 (statestore-test+0x18fbe96) #9 impala::StatestoreSslTest_ValidCertSmokeTest_Test::TestBody() /home/systest/Impala/be/src/statestore/statestore-test.cc:110:3 (statestore-test+0x18fc638) #10 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (statestore-test+0x4105212) #11 __libc_start_main /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x2082f){code} > Thrift server shutdown is racey > ------------------------------- > > Key: IMPALA-9314 > URL: https://issues.apache.org/jira/browse/IMPALA-9314 > Project: IMPALA > Issue Type: Sub-task > Components: Backend > Reporter: Sahil Takiar > Priority: Major > > There are multiple issues with shutting down the {{ThriftServer}}: > * The docs for {{ThriftServer}} say "TODO: shutdown is buggy (which only > harms tests)" > * TSAN reports a thread leak when using the {{ThriftServer}} in > {{statestore-test}} > ** Attempts to shutdown the {{ThriftServer}} in the {{Statestore}} don't > seem to work properly > * The {{TAcceptQueueServer}} actually uses a {{volatile}} {{boolean}} called > {{stop_}} to coordinate the shutdown (see {{TAcceptQueueServer::stop()}}) > which is not-thread safe (TSAN complains about this as well) > According to the docs this only affects test, specifically > {{statestore-test}}. We should consider whether it is worth fixing the > shutdown logic in the {{ThriftServer}}. > {code:java} > WARNING: ThreadSanitizer: thread leak (pid=67752) > Thread T39 (tid=67794, finished) created by main thread at: > #0 pthread_create > /mnt/source/llvm/llvm-5.0.1.src-p1/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889 > (statestore-test+0x18903bb) > #1 boost::thread::start_thread_noexcept() <null> > (statestore-test+0x2f30b59) > #2 boost::thread::thread<void (*)(std::string const&, std::string const&, > boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, > boost::function<void ()>, impala::ThreadDebugInfo*, impala: > :Promise<long, (impala::PromiseMode)0>*>(void (*)(std::string const&, > std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*, > impala::Promise<long, (impala::PromiseMode)0>*), std::string, std::string, > boost::function<void ()>, impala::ThreadDebugIn > fo*, impala::Promise<long, (impala::PromiseMode)0>*) > /home/systest/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:419:13 > (statestore-test+0x1ebf8eb) > #3 impala::Thread::StartThread(std::string const&, std::string const&, > boost::function<void ()> const&, std::unique_ptr<impala::Thread, > std::default_delete<impala::Thread> >*, bool) > /home/systest/Impala/be/src/util/thread.cc:317:13 (statestore-test+0x1ebcbd5) > #4 impala::Status impala::Thread::Create<void > (impala::ThriftServer::ThriftServerEventProcessor::*)(), > impala::ThriftServer::ThriftServerEventProcessor*>(std::string const&, > std::string const&, void (impala::ThriftServer::ThriftServerEventProcessor::* > const&)(), imp > ala::ThriftServer::ThriftServerEventProcessor* const&, > std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) > /home/systest/Impala/be/src/util/thread.h:81:12 (statestore-test+0x2707497) > #5 > impala::ThriftServer::ThriftServerEventProcessor::StartAndWaitForServer() > /home/systest/Impala/be/src/rpc/thrift-server.cc:116:43 > (statestore-test+0x27046f2) > #6 impala::ThriftServer::Start() > /home/systest/Impala/be/src/rpc/thrift-server.cc:447:60 > (statestore-test+0x27063cc) > #7 impala::Statestore::Init(int) > /home/systest/Impala/be/src/statestore/statestore.cc:478:59 > (statestore-test+0x1d7a431) > #8 impala::StatestoreTest_SmokeTest_Test::TestBody() > /home/systest/Impala/be/src/statestore/statestore-test.cc:55:132 > (statestore-test+0x18fb381) > #9 void > testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, > void>(testing::Test*, void (testing::Test::*)(), char const*) <null> > (statestore-test+0x41051e2) > #10 __libc_start_main > /build/glibc-LK5gWL/glibc-2.23/csu/../csu/libc-start.c:291 > (libc.so.6+0x2082f){code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-all-unsubscr...@impala.apache.org For additional commands, e-mail: issues-all-h...@impala.apache.org