[ https://issues.apache.org/jira/browse/KUDU-2916?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Adar Dembo resolved KUDU-2916. ------------------------------ Resolution: Duplicate Fix Version/s: n/a > Admin.TestDumpMemTrackers is flaky in tsan > ------------------------------------------ > > Key: KUDU-2916 > URL: https://issues.apache.org/jira/browse/KUDU-2916 > Project: Kudu > Issue Type: Bug > Affects Versions: 1.10.0 > Reporter: Hao Hao > Priority: Major > Fix For: n/a > > > I saw a tsan failure for AdminCliTest.TestDumpMemTrackers with the following > log: > {noformat} > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-admin-test.cc:2162: > Failure > Value of: s.ok() > Actual: false > Expected: true > Runtime error: /tmp/dist-test-taskUWtx7r/build/tsan/bin/kudu: process exited > with non-zero status 66 > stdout: > {"id":"root","limit":-1,"current_consumption":481,"peak_consumption":481,"child_trackers":[{"id":"server","parent_id":"root","limit":-1,"current_consumption":313,"peak_consumption":313,"child_trackers":[{"id":"result-tracker","parent_id":"server","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_block_manager","parent_id":"server","limit":-1,"current_consumption":48,"peak_consumption":48},{"id":"tablet-9276b163452b4b0399ff2cae579f7251","parent_id":"server","limit":-1,"current_consumption":265,"peak_consumption":265,"child_trackers":[{"id":"DeltaMemStores","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"MemRowSet-0","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":265,"peak_consumption":265},{"id":"txn_tracker","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":67108864,"current_consumption":0,"peak_consumption":0}]}]},{"id":"ttl-cache-sharded_fifo_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"code_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"block_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"lbm-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_cache","parent_id":"root","limit":1073741824,"current_consumption":168,"peak_consumption":168,"child_trackers":[{"id":"log_cache:457a3168758d4f4f8f4c59e8dd179cd3:9276b163452b4b0399ff2cae579f7251","parent_id":"log_cache","limit":10485760,"current_consumption":168,"peak_consumption":168}]}]} > stderr: W0803 14:03:00.206982 8443 flags.cc:404] Enabled unsafe flag: > --never_fsync=true > W0803 14:03:00.849385 8443 thread.cc:599] rpc reactor (reactor) Time spent > creating pthread: real 0.590s user 0.230s sys 0.360s > W0803 14:03:00.849658 8443 thread.cc:566] rpc reactor (reactor) Time spent > starting thread: real 0.591s user 0.230s sys 0.360s > ================== > WARNING: ThreadSanitizer: destroy of a locked mutex (pid=8443) > #0 pthread_rwlock_destroy > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1313 > (kudu+0x4bbb24) > #1 glog_internal_namespace_::Mutex::~Mutex() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30 > (libglog.so.0+0x16488) > #2 cxa_at_exit_wrapper(void*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:386 > (kudu+0x48beb3) > and: > #0 operator new(unsigned long) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:57 > (kudu+0x52ae83) > #1 __allocate > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/new:228:10 > (libc++.so.1+0xd63f3) > #2 allocate > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1793 > (libc++.so.1+0xd63f3) > #3 allocate > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1547 > (libc++.so.1+0xd63f3) > #4 __init > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1591 > (libc++.so.1+0xd63f3) > #5 std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >::basic_string(std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1653 > (libc++.so.1+0xd63f3) > #6 std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > >::pair(std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/utility:324:5 > (libprotobuf.so.14+0x188711) > #7 void > std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*> > >::construct<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> >, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > const&>(std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> >*, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1805 > (libprotobuf.so.14+0x188711) > #8 void > std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*> > > >::__construct<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> >, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > const&>(std::__1::integral_constant<bool, true>, > std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*> >&, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1715 > (libprotobuf.so.14+0x188711) > #9 void > std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*> > > >::construct<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> >, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > const&>(std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*> >&, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1561 > (libprotobuf.so.14+0x188711) > #10 > std::__1::unique_ptr<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*>, > std::__1::__tree_node_destructor<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, void*> > > > > std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> >, std::__1::less<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > >, true>, > std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> > > > >::__construct_node<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > > const&>(std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2192 > (libprotobuf.so.14+0x188711) > #11 > std::__1::pair<std::__1::__tree_iterator<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int> >, > std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> >, void*>*, long>, bool> > std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> >, std::__1::less<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > >, true>, > std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void > const*, int> > > >::__emplace_unique_key_args<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > const&>(std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const&, > std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2137 > (libprotobuf.so.14+0x188711) > #12 bool > google::protobuf::InsertIfNotPresent<std::__1::map<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > >, > std::__1::allocator<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > > > > >(std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::pair<void const*, int>, > std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > >, > std::__1::allocator<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > > >*, > std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::pair<void const*, int>, > std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > >, > std::__1::allocator<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > > >::value_type::first_type const&, > std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::pair<void const*, int>, > std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > >, > std::__1::allocator<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::pair<void const*, int> > > >::value_type::second_type const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:1273:16 > (libprotobuf.so.14+0x184c18) > #13 > google::protobuf::SimpleDescriptorDatabase::DescriptorIndex<std::__1::pair<void > const*, int> >::AddFile(google::protobuf::FileDescriptorProto const&, > std::__1::pair<void const*, int>) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:57 > (libprotobuf.so.14+0x184c18) > #14 google::protobuf::EncodedDescriptorDatabase::Add(void const*, int) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:316:19 > (libprotobuf.so.14+0x182d7c) > #15 google::protobuf::DescriptorPool::InternalAddGeneratedFile(void > const*, int) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor.cc:1315:3 > (libprotobuf.so.14+0x13b705) > #16 > google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::(anonymous > namespace)::AddDescriptorsImpl() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:240:3 > (libprotobuf.so.14+0x237c10) > #17 google::protobuf::internal::FunctionClosure0::Run() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5 > (libprotobuf.so.14+0xd330b) > #18 google::protobuf::GoogleOnceInitImpl(long*, > google::protobuf::Closure*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14 > (libprotobuf.so.14+0xd5d6a) > #19 google::protobuf::GoogleOnceInit(long*, void (*)()) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5 > (libprotobuf.so.14+0x237b66) > #20 > google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:251 > (libprotobuf.so.14+0x237b66) > #21 > google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::(anonymous > namespace)::AddDescriptorsImpl() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:179:3 > (libprotobuf.so.14+0x12ca1d) > #22 google::protobuf::internal::FunctionClosure0::Run() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5 > (libprotobuf.so.14+0xd330b) > #23 google::protobuf::GoogleOnceInitImpl(long*, > google::protobuf::Closure*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14 > (libprotobuf.so.14+0xd5d6a) > #24 google::protobuf::GoogleOnceInit(long*, void (*)()) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5 > (libprotobuf.so.14+0xcf0a6) > #25 > google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::AddDescriptors() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:185 > (libprotobuf.so.14+0xcf0a6) > #26 > google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::StaticDescriptorInitializer::StaticDescriptorInitializer() > > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:190 > (libprotobuf.so.14+0xcf0a6) > #27 __cxx_global_var_init > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:192 > (libprotobuf.so.14+0xcf0a6) > #28 _GLOBAL__sub_I_api.pb.cc > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc > (libprotobuf.so.14+0xcf0a6) > #29 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78 > (ld-linux-x86-64.so.2+0x102d9) > Location is global 'google::vmodule_lock' of size 64 at 0x7f67b632f878 > (libglog.so.0+0x00000023c878) > Mutex M134 (0x7f67b632f878) created at: > #0 pthread_rwlock_init > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1304 > (kudu+0x4bbd94) > #1 glog_internal_namespace_::Mutex::Mutex() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:247:19 > (libglog.so.0+0xcca6) > #2 __cxx_global_var_init.15 > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:120 > (libglog.so.0+0xcca6) > #3 _GLOBAL__sub_I_vlog_is_on.cc > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc > (libglog.so.0+0xcca6) > #4 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78 > (ld-linux-x86-64.so.2+0x102d9) > SUMMARY: ThreadSanitizer: destroy of a locked mutex > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30 > in glog_internal_namespace_::Mutex::~Mutex() > ================== > ================== > WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=8443) > Cycle in lock order graph: M1860 (0x7b14000167b8) => M37857730294380664 > (0x000000000000) => M1860 > Mutex M37857730294380664 acquired here while holding mutex M1860 in main > thread: > #0 pthread_rwlock_wrlock > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1352 > (kudu+0x4bb154) > #1 glog_internal_namespace_::Mutex::Lock() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30 > (libglog.so.0+0x1abb7) > #2 > glog_internal_namespace_::MutexLock::MutexLock(glog_internal_namespace_::Mutex*) > > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:290 > (libglog.so.0+0x1abb7) > #3 google::InitVLOG3__(int**, int*, char const*, int) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:199 > (libglog.so.0+0x1abb7) > #4 > kudu::rpc::Messenger::ShutdownInternal(kudu::rpc::Messenger::ShutdownMode) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:283:5 > (libkrpc.so+0xa9471) > #5 kudu::rpc::Messenger::AllExternalReferencesDropped() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:249:3 > (libkrpc.so+0xa9227) > #6 std::__1::mem_fun_t<void, > kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1120:17 > (libkrpc.so+0xad755) > #7 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*, > std::__1::mem_fun_t<void, kudu::rpc::Messenger>, > std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586 > (libkrpc.so+0xad755) > #8 std::__1::__shared_count::__release_shared() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9 > (kudu+0x576f2e) > #9 std::__1::__shared_weak_count::__release_shared() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532 > (kudu+0x576f2e) > #10 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468 > (kudu+0x576f2e) > #11 kudu::rpc::Proxy::~Proxy() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/proxy.cc:72:1 > (libkrpc.so+0xbd0f3) > #12 kudu::server::GenericServiceProxy::~GenericServiceProxy() > /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/server/server_base.proxy.cc:26:1 > (libserver_base_proto.so+0x4bd79) > #13 > std::__1::default_delete<kudu::server::GenericServiceProxy>::operator()(kudu::server::GenericServiceProxy*) > const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5 > (libkudu_tools_util.so+0x74acf) > #14 std::__1::unique_ptr<kudu::server::GenericServiceProxy, > std::__1::default_delete<kudu::server::GenericServiceProxy> > >::reset(kudu::server::GenericServiceProxy*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2598 > (libkudu_tools_util.so+0x74acf) > #15 std::__1::unique_ptr<kudu::server::GenericServiceProxy, > std::__1::default_delete<kudu::server::GenericServiceProxy> >::~unique_ptr() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2552 > (libkudu_tools_util.so+0x74acf) > #16 kudu::tools::DumpMemTrackers(std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned > short) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:629 > (libkudu_tools_util.so+0x74acf) > #17 kudu::tools::(anonymous > namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10 > (kudu+0x5b8599) > #18 > _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_ > > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 > (kudu+0x53956b) > #19 kudu::Status > std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status > (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext > const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), > kudu::tools::RunnerContext const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318 > (kudu+0x53956b) > #20 std::__1::__function::__func<kudu::Status > (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status > (*)(kudu::tools::RunnerContext const&)>, kudu::Status > (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 > (kudu+0x53949d) > #21 std::__1::function<kudu::Status (kudu::tools::RunnerContext > const&)>::operator()(kudu::tools::RunnerContext const&) const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 > (libkudu_tools_util.so+0x6b2a4) > #22 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, > std::__1::allocator<kudu::tools::Mode*> > const&, > std::__1::unordered_map<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > >, > std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > >, > std::__1::allocator<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > > > > const&, > std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10 > (libkudu_tools_util.so+0x69974) > #23 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, > std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, > std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15 > (kudu+0x5bbcf6) > #24 kudu::tools::RunTool(int, char**, bool) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16 > (kudu+0x5bcc51) > #25 main > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10 > (kudu+0x5bcfbe) > Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative > warning message > Mutex M1860 acquired here while holding mutex M37857730294380664 in thread > T7: > #0 AnnotateRWLockAcquired > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc:271 > (kudu+0x4dbd0c) > #1 kudu::rw_spinlock::lock() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:112:5 > (libkudu_client.so+0x16d842) > #2 kudu::percpu_rwlock::lock() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:222:22 > (libkudu_client.so+0x16d7d2) > #3 > std::__1::lock_guard<kudu::percpu_rwlock>::lock_guard(kudu::percpu_rwlock&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__mutex_base:104:27 > (libkrpc.so+0xaad39) > #4 kudu::rpc::Messenger::~Messenger() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:430 > (libkrpc.so+0xaad39) > #5 > std::__1::default_delete<kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) > const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5 > (libkrpc.so+0xb081b) > #6 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*, > std::__1::default_delete<kudu::rpc::Messenger>, > std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586 > (libkrpc.so+0xb081b) > #7 std::__1::__shared_count::__release_shared() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9 > (kudu+0x576f2e) > #8 std::__1::__shared_weak_count::__release_shared() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532 > (kudu+0x576f2e) > #9 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468 > (kudu+0x576f2e) > #10 std::__1::shared_ptr<kudu::rpc::Messenger>::reset() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4603:5 > (libkrpc.so+0xbf151) > #11 kudu::rpc::ReactorThread::RunThread() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:499 > (libkrpc.so+0xbf151) > #12 boost::_mfi::mf0<void, > kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 > (libkrpc.so+0xc9539) > #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, > boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, > kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 > (libkrpc.so+0xc948a) > #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, > kudu::rpc::ReactorThread>, > boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > > >::operator()() > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 > (libkrpc.so+0xc9413) > #15 > boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, > boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, > boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, > void>::invoke(boost::detail::function::function_buffer&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 > (libkrpc.so+0xc9209) > #16 boost::function0<void>::operator()() const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 > (libkrpc.so+0xb8a11) > #17 kudu::Thread::SuperviseThread(void*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:657:3 > (libkudu_util.so+0x1e1824) > Thread T7 'rpc reactor-845' (tid=8451, running) created by main thread at: > #0 pthread_create > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992 > (kudu+0x4bd8eb) > #1 kudu::Thread::StartThread(std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const&, > std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned > long, scoped_refptr<kudu::Thread>*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:601:15 > (libkudu_util.so+0x1e100b) > #2 kudu::Status kudu::Thread::Create<void > (kudu::rpc::ReactorThread::*)(), > kudu::rpc::ReactorThread*>(std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const&, > std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* > const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12 > (libkrpc.so+0xc48e5) > #3 kudu::rpc::ReactorThread::Init() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:185:10 > (libkrpc.so+0xbec4e) > #4 kudu::rpc::Reactor::Init() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:759:18 > (libkrpc.so+0xc32f1) > #5 kudu::rpc::Messenger::Init() > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:446:5 > (libkrpc.so+0xa90e2) > #6 > kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) > > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:205:3 > (libkrpc.so+0xa8b3d) > #7 kudu::Status > kudu::tools::BuildProxy<kudu::server::GenericServiceProxy>(std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned > short, std::__1::unique_ptr<kudu::server::GenericServiceProxy, > std::__1::default_delete<kudu::server::GenericServiceProxy> >*) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:328:3 > (libkudu_tools_util.so+0x7c55e) > #8 kudu::tools::DumpMemTrackers(std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned > short) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:591:3 > (libkudu_tools_util.so+0x747e3) > #9 kudu::tools::(anonymous > namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10 > (kudu+0x5b8599) > #10 > _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_ > > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 > (kudu+0x53956b) > #11 kudu::Status > std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status > (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext > const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), > kudu::tools::RunnerContext const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318 > (kudu+0x53956b) > #12 std::__1::__function::__func<kudu::Status > (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status > (*)(kudu::tools::RunnerContext const&)>, kudu::Status > (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 > (kudu+0x53949d) > #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext > const&)>::operator()(kudu::tools::RunnerContext const&) const > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 > (libkudu_tools_util.so+0x6b2a4) > #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, > std::__1::allocator<kudu::tools::Mode*> > const&, > std::__1::unordered_map<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> >, > std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > >, > std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > >, > std::__1::allocator<std::__1::pair<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > const, > std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> > > > > const&, > std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10 > (libkudu_tools_util.so+0x69974) > #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, > std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, > std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, > std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, > std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15 > (kudu+0x5bbcf6) > #16 kudu::tools::RunTool(int, char**, bool) > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16 > (kudu+0x5bcc51) > #17 main > /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10 > (kudu+0x5bcfbe) > SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) > /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30 > in glog_internal_namespace_::Mutex::Lock() > ================== > ThreadSanitizer: reported 2 warnings > {noformat} -- This message was sent by Atlassian JIRA (v7.6.14#76016)