[
https://issues.apache.org/jira/browse/IMPALA-6144?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sailesh Mukil resolved IMPALA-6144.
-----------------------------------
Resolution: Fixed
Fix Version/s: Impala 2.11.0
Commit in:
https://github.com/apache/incubator-impala/commit/e6c3a01b948777affaee3dd004a526f5e4dcd5a0
> Coordinator threads that publish RuntimeFilters continue to run after query
> failure/cancellation
> ------------------------------------------------------------------------------------------------
>
> Key: IMPALA-6144
> URL: https://issues.apache.org/jira/browse/IMPALA-6144
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Reporter: Mostafa Mokhtar
> Assignee: Sailesh Mukil
> Labels: memory, runtime-filters
> Fix For: Impala 2.11.0
>
>
> When running highly concurrent queries with lots of joins on a large cluster
> I noticed that there is lots of untracked memory on the coordinator node.
> Experiments showed that the majority of the untracked memory is coming from
> RunTimeFilters.
> After canceling the concurrent queries the Queries tab showed 0 "Running" and
> 0 "Waiting to be cancelled" queries while the memz tab show that memory
> allocated for the cancelled/completed queries is still there.
> Same issue happens if remote fragments finish while the coordinator node is
> in the progress of sending the runtime filters to the remote nodes.
> {code}
> CPU Time
> 1 of 1: 100.0% (20.348s of 20.348s)
> impalad ! apache::thrift::transport::TSSLSocket::write - [unknown source file]
> impalad ! apache::thrift::transport::TTransport::write + 0xc -
> TTransport.h:158
> impalad ! apache::thrift::transport::TSaslTransport::write + 0x3b -
> TSaslTransport.cpp:233
> impalad ! apache::thrift::transport::TBufferedTransport::writeSlow + 0x58 -
> [unknown source file]
> impalad ! apache::thrift::transport::TTransport::write + 0xb -
> TTransport.h:158
> impalad ! writeString<std::basic_string<char> > + 0x39 -
> TBinaryProtocol.tcc:186
> impalad !
> apache::thrift::protocol::TBinaryProtocolT<apache::thrift::transport::TTransport>::writeBinary
> - TBinaryProtocol.tcc:193
> impalad !
> apache::thrift::protocol::TVirtualProtocol<apache::thrift::protocol::TBinaryProtocolT<apache::thrift::transport::TTransport>,
> apache::thrift::protocol::TProtocolDefaults>::writeBinary_virt + 0xb -
> TVirtualProtocol.h:421
> impalad ! apache::thrift::protocol::TProtocol::writeBinary + 0x10 -
> TProtocol.h:468
> impalad ! impala::TBloomFilter::write + 0x72 -
> ImpalaInternalService_types.cpp:4397
> impalad ! impala::TPublishFilterParams::write + 0xad -
> ImpalaInternalService_types.cpp:4689
> impalad ! impala::ImpalaInternalService_PublishFilter_pargs::write + 0x44 -
> ImpalaInternalService.cpp:903
> impalad ! impala::ImpalaInternalServiceClient::send_PublishFilter + 0x78 -
> ImpalaInternalService.cpp:1310
> impalad ! impala::ImpalaBackendClient::PublishFilter + 0x14 -
> backend-client.h:117
> impalad ! impala::ClientConnection<impala::ImpalaBackendClient>::DoRpc<void
> (impala::TPublishFilterResult&, impala::TPublishFilterParams const&, bool*)
> impala::ImpalaBackendClient::*, impala::TPublishFilterParams,
> impala::TPublishFilterResult> + 0x57 - client-cache.h:240
> impalad ! impala::Coordinator::BackendState::PublishFilter + 0x288 -
> coordinator-backend-state.cc:415
> impalad ! impala::Coordinator::UpdateFilter + 0x5ff - coordinator.cc:1167
> impalad ! impala::ImpalaServer::UpdateFilter + 0x32 - impala-server.cc:2089
> impalad ! impala::ImpalaInternalServiceProcessor::process_UpdateFilter +
> 0x1e2 - ImpalaInternalService.cpp:1613
> impalad ! impala::ImpalaInternalServiceProcessor::dispatchCall + 0xe8 -
> ImpalaInternalService.cpp:1370
> impalad ! apache::thrift::TDispatchProcessor::process + 0xab -
> TDispatchProcessor.h:121
> impalad ! apache::thrift::server::TAcceptQueueServer::Task::run + 0x15e -
> TAcceptQueueServer.cpp:77
> impalad ! impala::ThriftThread::RunRunnable + 0x68 - thrift-thread.cc:74
> impalad ! boost::_mfi::mf2<void, impala::ThriftThread,
> boost::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<unsigned long>*>::operator() + 0x3a - mem_fn_template.hpp:280
> impalad ! operator()<boost::_mfi::mf2<void, impala::ThriftThread,
> boost::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<long unsigned int>*>, boost::_bi::list0> + 0x1e - bind.hpp:392
> impalad ! boost::_bi::bind_t<void, boost::_mfi::mf2<void,
> impala::ThriftThread,
> boost::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<unsigned long>*>,
> boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
> boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable>>,
> boost::_bi::value<impala::Promise<unsigned long>*>>>::operator() -
> bind_template.hpp:20
> impalad !
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf2<void, impala::ThriftThread,
> boost::shared_ptr<apache::thrift::concurrency::Runnable>,
> impala::Promise<unsigned long>*>,
> boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>,
> boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable>>,
> boost::_bi::value<impala::Promise<unsigned long>*>>>, void>::invoke + 0x9 -
> function_template.hpp:153
> impalad ! boost::function0<void>::operator() + 0x1a -
> function_template.hpp:767
> impalad ! impala::Thread::SuperviseThread + 0x1a7 - thread.cc:352
> impalad ! operator()<void (*)(const std::basic_string<char>&, const
> std::basic_string<char>&, boost::function<void()>, impala::Promise<long
> int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
> impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string
> const&, boost::function<void (void)>, impala::Promise<long>*),
> boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() -
> bind_template.hpp:20
> impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void
> (*)(std::string const&, std::string const&, boost::function<void (void)>,
> impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 -
> thread.hpp:116
> impalad ! thread_proxy + 0xd9 - [unknown source file]
> libpthread-2.12.so ! start_thread + 0xd0 - [unknown source file]
> libc-2.12.so ! clone + 0x6c - [unknown source file]
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
