[
https://issues.apache.org/jira/browse/KUDU-2480?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Andrew Wong resolved KUDU-2480.
-------------------------------
Resolution: Fixed
Fix Version/s: 1.8.0
Dan merged this as 4479c20.
> tsan failure of master-stress-test
> ----------------------------------
>
> Key: KUDU-2480
> URL: https://issues.apache.org/jira/browse/KUDU-2480
> Project: Kudu
> Issue Type: Test
> Reporter: Hao Hao
> Assignee: Dan Burkert
> Priority: Major
> Fix For: 1.8.0
>
> Attachments: master-stress-test.txt
>
>
> master-stress-test recently has been very flaky(~24%). One of the failure log
> {noformat}WARNING: ThreadSanitizer: data race (pid=26513)
> Read of size 8 at 0x7ffb5e5b88b8 by thread T65:
> #0 kudu::Status::operator=(kudu::Status const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/status.h:469:7
> (libmaster.so+0x10bd00)
> #1 kudu::Synchronizer::StatusCB(kudu::Status const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/async_util.h:44:8
> (libmaster.so+0x10bc40)
> #2 kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status
> const&)>::Run(kudu::Synchronizer*, kudu::Status const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:192:12
> (libmaster.so+0x10c708)
> #3 kudu::internal::InvokeHelper<false, void,
> kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status
> const&)>, void ()(kudu::Synchronizer*, kudu::Status
> const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
> (kudu::Synchronizer::*)(kudu::Status const&)>, kudu::Synchronizer*,
> kudu::Status const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:889:14
> (libmaster.so+0x10c5e8)
> #4 kudu::internal::Invoker<1,
> kudu::internal::BindState<kudu::internal::RunnableAdapter<void
> (kudu::Synchronizer::*)(kudu::Status const&)>, void ()(kudu::Synchronizer*,
> kudu::Status const&), void
> ()(kudu::internal::UnretainedWrapper<kudu::Synchronizer>)>, void
> ()(kudu::Synchronizer*, kudu::Status
> const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1118:12
> (libmaster.so+0x10c51a)
> #5 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&)
> const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:436:12
> (libmaster.so+0x10b831)
> #6 kudu::master::HmsNotificationLogListenerTask::RunLoop()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/hms_notification_log_listener.cc:136:10
> (libmaster.so+0x108e0a)
> #7 boost::_mfi::mf0<void,
> kudu::master::HmsNotificationLogListenerTask>::operator()(kudu::master::HmsNotificationLogListenerTask*)
> const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29
> (libmaster.so+0x110ea9)
> #8 void
> boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*>
> >::operator()<boost::_mfi::mf0<void,
> kudu::master::HmsNotificationLogListenerTask>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::master::HmsNotificationLogListenerTask>&, boost::_bi::list0&, int)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9
> (libmaster.so+0x110dfa)
> #9 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::master::HmsNotificationLogListenerTask>,
> boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*>
> > >::operator()()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libmaster.so+0x110d83)
> #10
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>,
> boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*>
> > >, void>::invoke(boost::detail::function::function_buffer&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libmaster.so+0x110b79)
> #11 boost::function0<void>::operator()() const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb64b1)
> #12 kudu::Thread::SuperviseThread(void*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:603:3
> (libkudu_util.so+0x1bd8b4)
> Previous write of size 8 at 0x7ffb5e5b88b8 by thread T24 (mutexes: read
> M1468):
> #0
> boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*>
> >::init(boost::intrusive::list_node<void*>* const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/circular_list_algorithms.hpp:72:22
> (libkrpc.so+0x99c92)
> #1
> boost::intrusive::generic_hook<boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*>
> >, boost::intrusive::dft_tag, (boost::intrusive::link_mode_type)1,
> (boost::intrusive::base_hook_type)1>::generic_hook()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/detail/generic_hook.hpp:174:10
> (libkrpc.so+0xc4669)
> #2 boost::intrusive::list_base_hook<void, void, void>::list_base_hook()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/list_hook.hpp:83:7
> (libkrpc.so+0xc2049)
> #3 kudu::rpc::ReactorTask::ReactorTask()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:262:14
> (libkrpc.so+0xbd5fb)
> #4
> kudu::rpc::QueueTransferTask::QueueTransferTask(gscoped_ptr<kudu::rpc::OutboundTransfer,
> kudu::DefaultDeleter<kudu::rpc::OutboundTransfer> >, kudu::rpc::Connection*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/connection.cc:432:3
> (libkrpc.so+0x98ea4)
> #5
> kudu::rpc::Connection::QueueResponseForCall(gscoped_ptr<kudu::rpc::InboundCall,
> kudu::DefaultDeleter<kudu::rpc::InboundCall> >)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/connection.cc:474:33
> (libkrpc.so+0x94d79)
> #6 kudu::rpc::InboundCall::Respond(google::protobuf::MessageLite const&,
> bool)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/inbound_call.cc:165:10
> (libkrpc.so+0xa11b9)
> #7 kudu::rpc::InboundCall::RespondSuccess(google::protobuf::MessageLite
> const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/inbound_call.cc:109:3
> (libkrpc.so+0xa0fd5)
> #8 kudu::rpc::RpcContext::RespondSuccess()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/rpc_context.cc:77:12
> (libkrpc.so+0xd5b35)
> #9
> kudu::master::MasterServiceImpl::AlterTable(kudu::master::AlterTableRequestPB
> const*, kudu::master::AlterTableResponsePB*, kudu::rpc::RpcContext*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_service.cc:350:8
> (libmaster.so+0x128903)
> #10
> kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity>
> const&, scoped_refptr<kudu::rpc::ResultTracker>
> const&)::$_13::operator()(google::protobuf::Message const*,
> google::protobuf::Message*, kudu::rpc::RpcContext*) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/master/master.service.cc:259:13
> (libmaster_proto.so+0xe2544)
> #11
> _ZNSt3__18__invokeIRZN4kudu6master15MasterServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE4$_13JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1
> (libmaster_proto.so+0xe24d1)
> #12
> _ZNSt3__128__invoke_void_return_wrapperIvE6__callIJRZN4kudu6master15MasterServiceIfC1ERK13scoped_refptrINS3_12MetricEntityEERKS6_INS3_3rpc13ResultTrackerEEE4$_13PKN6google8protobuf7MessageEPSK_PNSB_10RpcContextEEEEvDpOT_
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:349
> (libmaster_proto.so+0xe24d1)
> #13
> std::__1::__function::__func<kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity>
> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13,
> std::__1::allocator<kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity>
> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13>, void
> ()(google::protobuf::Message const*, google::protobuf::Message*,
> kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&,
> google::protobuf::Message*&&, kudu::rpc::RpcContext*&&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12
> (libmaster_proto.so+0xe23e4)
> #14 std::__1::function<void ()(google::protobuf::Message const*,
> google::protobuf::Message*,
> kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*,
> google::protobuf::Message*, kudu::rpc::RpcContext*) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12
> (libkrpc.so+0xefce9)
> #15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_if.cc:145:3
> (libkrpc.so+0xef632)
> #16 kudu::rpc::ServicePool::RunThread()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:225:15
> (libkrpc.so+0xf0b8c)
> #17 boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29
> (libkrpc.so+0xf2a69)
> #18 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ServicePool>&, boost::_bi::list0&, int)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9
> (libkrpc.so+0xf29ba)
> #19 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >
> >::operator()()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libkrpc.so+0xf2943)
> #20
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ServicePool>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libkrpc.so+0xf2739)
> #21 boost::function0<void>::operator()() const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb64b1)
> #22 kudu::Thread::SuperviseThread(void*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:603:3
> (libkudu_util.so+0x1bd8b4)
> Location is stack of thread T24.
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
