[
https://issues.apache.org/jira/browse/IMPALA-4062?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Tim Armstrong updated IMPALA-4062:
----------------------------------
Issue Type: Improvement (was: Bug)
> Create thread pool for HdfsScanNode::ScannerThread to limit Kernel contention
> -----------------------------------------------------------------------------
>
> Key: IMPALA-4062
> URL: https://issues.apache.org/jira/browse/IMPALA-4062
> Project: IMPALA
> Issue Type: Improvement
> Components: Backend
> Affects Versions: Impala 2.6.0
> Reporter: Mostafa Mokhtar
> Priority: Major
> Labels: performance, scalability
> Attachments: TPC-DS Q27.txt, q27_perf_kernel.txt, q_27_spinning_1.zip
>
>
> Servers with modern processors like E5-2698 can have up to 80 logical
> processors per server, as a result queries occasionally end up running with a
> significantly larger number of threads.
> Creating and destroying threads is expensive and wastes lots of resources,
> hence consider creating a thread pool for scanner threads to avoid resource
> contention during thread creation.
> For TPC-DS Q27 >40% of CPU cycles are spent pthread_mutex_unlock and
> pthread_mutex_lock
> Call stacks
> {code}
> CPU Time
> 1 of 5: 71.4% (31.928s of 44.725s)
> impalad ! pthread_mutex_unlock - mutex.hpp
> impalad ! boost::mutex::unlock + 0x10 - mutex.hpp:125
> impalad ! ~unique_lock + 0x16 - lock_types.hpp:331
> impalad ! impala::HdfsScanNode::ScannerThread + 0x2aa - hdfs-scan-node.cc:1044
> impalad ! boost::function0<void>::operator() + 0x1a -
> function_template.hpp:767
> impalad ! impala::Thread::SuperviseThread + 0x20e - thread.cc:318
> impalad ! operator()<void (*)(const std::basic_string<char>&, const
> std::basic_string<char>&, boost::function<void()>, impala::Promise<long
> int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
> impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string
> const&, boost::function<void (void)>, impala::Promise<long>*),
> boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() -
> bind_template.hpp:20
> impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void
> (*)(std::string const&, std::string const&, boost::function<void (void)>,
> impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 -
> thread.hpp:116
> impalad ! thread_proxy + 0xd9 - [unknown source file]
> libpthread.so.0 ! start_thread + 0xd0 - [unknown source file]
> libc.so.6 ! clone + 0x6c - [unknown source file]
> {code}
> {code}
> CPU Time
> 2 of 5: 26.4% (11.787s of 44.725s)
> impalad ! pthread_mutex_unlock - mutex.hpp
> impalad ! boost::mutex::unlock + 0x10 - mutex.hpp:125
> impalad ! ~unique_lock + 0x16 - lock_types.hpp:331
> impalad ! impala::Promise<bool>::Get + 0x82d - promise.h:94
> impalad ! impala::CountingBarrier::Wait - counting-barrier.h:42
> impalad ! impala::HdfsScanNode::ScannerThread + 0x2aa - hdfs-scan-node.cc:1044
> impalad ! boost::function0<void>::operator() + 0x1a -
> function_template.hpp:767
> impalad ! impala::Thread::SuperviseThread + 0x20e - thread.cc:318
> impalad ! operator()<void (*)(const std::basic_string<char>&, const
> std::basic_string<char>&, boost::function<void()>, impala::Promise<long
> int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
> impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string
> const&, boost::function<void (void)>, impala::Promise<long>*),
> boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() -
> bind_template.hpp:20
> impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void
> (*)(std::string const&, std::string const&, boost::function<void (void)>,
> impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 -
> thread.hpp:116
> impalad ! thread_proxy + 0xd9 - [unknown source file]
> libpthread.so.0 ! start_thread + 0xd0 - [unknown source file]
> libc.so.6 ! clone + 0x6c - [unknown source file]
> {code}
> {code}
> CPU Time
> 1 of 11: 98.4% (35.681s of 36.271s)
> impalad ! pthread_mutex_lock - mutex.hpp
> impalad ! boost::mutex::lock + 0x10 - mutex.hpp:116
> impalad ! [impalad] + 0x25f9daf - [unknown source file]
> impalad ! boost::function0<void>::operator() + 0x1a -
> function_template.hpp:767
> impalad ! impala::Thread::SuperviseThread + 0x20e - thread.cc:318
> impalad ! operator()<void (*)(const std::basic_string<char>&, const
> std::basic_string<char>&, boost::function<void()>, impala::Promise<long
> int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
> impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string
> const&, boost::function<void (void)>, impala::Promise<long>*),
> boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() -
> bind_template.hpp:20
> impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void
> (*)(std::string const&, std::string const&, boost::function<void (void)>,
> impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>,
> boost::_bi::value<std::string>, boost::_bi::value<boost::function<void
> (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 -
> thread.hpp:116
> impalad ! thread_proxy + 0xd9 - [unknown source file]
> libpthread.so.0 ! start_thread + 0xd0 - [unknown source file]
> libc.so.6 ! clone + 0x6c - [unknown source file]
> {code}
> Top functions from Perf
> {code}
> Samples: 2M of event 'cycles', Event count (approx.): 298397618324
> 46.59% [kernel] [k] _spin_lock
> 4.78% impalad [.]
> impala::ScalarColumnReader<impala::DecimalValue<int>,
> true>::ReadNonRepeatedValueBatch(impala::MemPool*, int, int, unsigned char*,
> int*)
> 4.49% impalad [.] bool
> impala::RleDecoder::Get<int>(int*)
> 3.11% impalad [.]
> snappy::RawUncompress(snappy::Source*, char*)
> 2.67% impalad [.]
> impala::ScalarColumnReader<int,
> true>::ReadNonRepeatedValueBatch(impala::MemPool*, int, int, unsigned char*,
> int*)
> 2.37% impalad [.]
> _ZNK6impala11BloomFilter14BucketFindAVX2Ejj.isra.102
> 2.12% impalad [.]
> impala::HdfsParquetScanner::AssembleRows(std::vector<impala::ParquetColumnReader*,
> std::allocator<impala::ParquetColumnReader*> > const&,
> 2.00% impalad [.]
> impala::HdfsParquetScanner::EvalRuntimeFilters(impala::TupleRow*)
> 1.71% libc-2.12.so [.] memcpy
> 1.47% impalad [.]
> impala::ParquetLevelDecoder::FillCache(int, int*)
> 1.26% libpthread-2.12.so [.] pthread_mutex_lock
> 1.23% [kernel] [k] futex_wait_setup
> 1.15% [kernel] [k] clear_page_c_e
> 0.92% [kernel] [k]
> copy_user_generic_string
> 0.92% [kernel] [k]
> smp_invalidate_interrupt
> 0.88% [kernel] [k] futex_wake
> 0.86% impalad [.]
> impala::ScalarColumnReader<long,
> true>::ReadNonRepeatedValueBatch(impala::MemPool*, int, int, unsigned char*,
> int*)
> 0.83% impalad [.]
> impala::HdfsScanNode::ScannerThread()
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
