Laszlo Gaal created IMPALA-13213:
------------------------------------
Summary: impalad crash in Coordinator::HandleExecStateTransition()
on ARM during exhaustive-release run
Key: IMPALA-13213
URL: https://issues.apache.org/jira/browse/IMPALA-13213
Project: IMPALA
Issue Type: Bug
Affects Versions: Impala 4.5.0
Reporter: Laszlo Gaal
The same test run was hit by multiple instance of IMPALA-12336 as well, so the
crash might just be a consequence of the memory exhaustion.
The memory tracker log showed {{AvailableMemory}} going as low as ~40 MB on a
test runner with 64GB of physical memory.
The log bundle reports the following stack trace from the core dump:
{code}
#0 0x0000ffffb4936274 in raise () from /lib64/libc.so.6
[Current thread is 1 (Thread 0xfffc0fa3ed80 (LWP 248075))]
To enable execution of this file add
add-auto-load-safe-path
/data0/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/lib64/libstdc++.so.6.0.28-gdb.py
line to your configuration file "/var/lib/jenkins/.config/gdb/gdbinit".
To completely disable this security protection add
set auto-load safe-path /
line to your configuration file "/var/lib/jenkins/.config/gdb/gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual. E.g., run from the shell:
info "(gdb)Auto-loading safe path"
#0 0x0000ffffb4936274 in raise () from /lib64/libc.so.6
#1 0x0000ffffb4920a2c in abort () from /lib64/libc.so.6
#2 0x0000ffffb4c29024 in __gnu_cxx::__verbose_terminate_handler () at
../../../../gcc-10.4.0/libstdc++-v3/libsupc++/vterminate.cc:50
#3 0x0000ffffb4c26b8c in __cxxabiv1::__terminate (handler=<optimized out>) at
../../../../gcc-10.4.0/libstdc++-v3/libsupc++/eh_terminate.cc:48
#4 0x0000ffffb4c26bf0 in std::terminate () at
../../../../gcc-10.4.0/libstdc++-v3/libsupc++/eh_terminate.cc:58
#5 0x0000ffffb4c26ee4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x425fd940,
tinfo=0x4bb3298 <typeinfo for boost::wrapexcept<std::out_of_range>>,
dest=0x13bd220 <boost::wrapexcept<std::out_of_range>::~wrapexcept()>) at
../../../../gcc-10.4.0/libstdc++-v3/libsupc++/eh_throw.cc:95
#6 0x00000000021ab900 in boost::throw_exception<std::out_of_range> (e=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/throw_exception.hpp:165
#7 boost::unordered::unordered_map<impala::Coordinator::ExecState, char
const*, boost::hash<impala::Coordinator::ExecState>,
std::equal_to<impala::Coordinator::ExecState>,
std::allocator<std::pair<impala::Coordinator::ExecState const, char const*> >
>::at (this=this@entry=0x4c5e350
<impala::Coordinator::HandleExecStateTransition(impala::Coordinator::ExecState,
impala::Coordinator::ExecState)::exec_state_to_event>, k=@0xfffc0fa3d62c:
impala::Coordinator::ExecState::CANCELLED) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/unordered/unordered_map.hpp:1900
#8 0x00000000021a28dc in impala::Coordinator::HandleExecStateTransition
(this=this@entry=0x31bc4000,
old_state=old_state@entry=impala::Coordinator::ExecState::EXECUTING,
new_state=<optimized out>,
new_state@entry=impala::Coordinator::ExecState::CANCELLED) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/coordinator.cc:865
#9 0x00000000021a305c in impala::Coordinator::SetNonErrorTerminalState
(this=this@entry=0x31bc4000,
state=state@entry=impala::Coordinator::ExecState::CANCELLED) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/coordinator.cc:798
#10 0x00000000021a3228 in impala::Coordinator::Cancel (this=0x31bc4000,
wait_until_finalized=true) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/coordinator.cc:1066
#11 0x000000000189fff4 in impala::ClientRequestState::Cancel
(this=this@entry=0x302000000, cause=cause@entry=0xfffc0fa3dd08,
wait_until_finalized=wait_until_finalized@entry=true) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/client-request-state.cc:1523
#12 0x00000000018a012c in impala::ClientRequestState::Finalize
(this=0x302000000, cause=cause@entry=0xfffc0fa3dd08) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/client-request-state.cc:1066
#13 0x00000000021cb118 in impala::QueryDriver::Finalize (this=0x41a9f270,
query_handle=query_handle@entry=0xfffc0fa3daa0,
check_inflight=check_inflight@entry=false, cause=cause@entry=0xfffc0fa3dd08) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/runtime/query-driver.h:50
#14 0x00000000016ded88 in impala::ImpalaServer::UnregisterQuery
(this=this@entry=0x325ef400, query_id=...,
check_inflight=check_inflight@entry=false, cause=cause@entry=0xfffc0fa3dd08) at
/data0/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/gcc-10.4.0/include/c++/10.4.0/bits/shared_ptr_base.h:1329
#15 0x00000000016df3bc in impala::ImpalaServer::UnregisterQueryDiscardResult
(this=this@entry=0x325ef400, query_id=...,
check_inflight=check_inflight@entry=false, cause=cause@entry=0xfffc0fa3dd08) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/impala-server.cc:1584
#16 0x00000000016e0394 in impala::ImpalaServer::CloseSessionInternal
(this=this@entry=0x325ef400, session_id=..., secret=...,
ignore_if_absent=ignore_if_absent@entry=true) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/impala-server.cc:1835
#17 0x00000000016e1390 in impala::ImpalaServer::ConnectionEnd (this=0x325ef400,
connection_context=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/service/impala-server.cc:2603
#18 0x00000000012bfd84 in
impala::ThriftServer::ThriftServerEventProcessor::deleteContext
(this=0x325cc780, context=<optimized out>, input=..., output=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/rpc/thrift-server.cc:256
#19 0x00000000012d5ccc in apache::thrift::server::TAcceptQueueServer::Task::run
(this=0x1e07929c0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/rpc/TAcceptQueueServer.cpp:125
#20 0x00000000012c87b8 in boost::_mfi::mf2<void, impala::ThriftThread,
std::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<std::thread::id, (impala::PromiseMode)0>*>::operator()
(this=0xd4cc8570, this=0xd4cc8570, a2=<optimized out>, a1=..., p=0x8b3f6be0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:280
#21 boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<std::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<std::thread::id, (impala::PromiseMode)0>*>
>::operator()<boost::_mfi::mf2<void, impala::ThriftThread,
std::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<std::thread::id, (impala::PromiseMode)0>*>, boost::_bi::list0>
(a=<synthetic pointer>..., f=..., this=0xd4cc8580) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:398
#22 boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread,
std::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<std::thread::id, (impala::PromiseMode)0>*>,
boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<std::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<std::thread::id, (impala::PromiseMode)0>*> >
>::operator() (this=0xd4cc8570) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294
#23
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf2<void, impala::ThriftThread,
std::shared_ptr<apache::thrift::concurrency::Runnable>,
impala::Promise<std::thread::id, (impala::PromiseMode)0>*>,
boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
boost::_bi::value<std::shared_ptr<apache::thrift::concurrency::Runnable> >,
boost::_bi::value<impala::Promise<std::thread::id, (impala::PromiseMode)0>*> >
>, void>::invoke (function_obj_ptr=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:158
#24 0x0000000001abb448 in boost::function0<void>::operator()
(this=0xfffc0fa3e500) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/function/function_template.hpp:763
#25 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*) (name=..., category=...,
functor=..., parent_thread_info=<optimized out>, thread_started=0xffff1563dca0)
at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/repos/Impala/be/src/util/thread.cc:360
#26 0x0000000001abc700 in
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void
(*&)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*), boost::_bi::list0&, int) (a=<synthetic pointer>...,
f=@0x7a68ad88: 0x1abb1c0
<impala::Thread::SuperviseThread(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*)>, this=0x7a68ad90) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:531
#27 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >
>::operator()() (this=0x7a68ad88) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/bind/bind.hpp:1294
#28 boost::detail::thread_data<boost::_bi::bind_t<void, void
(*)(std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, boost::function<void
()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > >,
boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run()
(this=0x7a68ac40) at
/data/jenkins/workspace/impala-asf-master-exhaustive-release-arm/Impala-Toolchain/toolchain-packages-gcc10.4.0/boost-1.74.0-p1/include/boost/thread/detail/thread.hpp:120
#29 0x0000000002306538 in thread_proxy ()
#30 0x0000ffffb69478b8 in start_thread () from /lib64/libpthread.so.0
#31 0x0000ffffb4923afc in thread_start () from /lib64/libc.so.6
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
