Hello,
I am seeing a deadlock when destructing an ObjectOutputStream. I have
attached the stack trace.
I did some debugging and found that the issue seems to be that the mutex in
question is already held by this thread (I checked the __owner field in the
pthread_mutex_t which points to the hanging thread)
Unfortunately the stack trace doesn’t show exactly which mutex it is trying
to lock. I wonder if someone more familiar with the IO code has some ideas
what might be the issue and where to dig deeper?
Appreciate the help,
Li
Thread 39 (Thread 0xe2199eee700 (LWP 1392) "python3.10"):
#0 __lll_lock_wait (futex=futex@entry=0xe2158016c60, private=0) at
lowlevellock.c:52
#1 0x00000e223fe14843 in __GI___pthread_mutex_lock (mutex=0xe2158016c60) at
../nptl/pthread_mutex_lock.c:80
#2 0x00000e223a4c7be3 in virtual thunk to arrow::fs::(anonymous
namespace)::ObjectOutputStream::Close() () at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/status.h:140
#3 0x00000e223993eaef in arrow::io::internal::CloseFromDestructor
(file=file@entry=0xe2158005c80) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/io/interfaces.cc:284
#4 0x00000e223a4b9250 in arrow::fs::(anonymous
namespace)::ObjectOutputStream::~ObjectOutputStream (this=0xe2158005b40,
__in_chrg=<optimized out>, __vtt_parm=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/filesystem/s3fs.cc:1398
#5 __gnu_cxx::new_allocator<arrow::fs::(anonymous
namespace)::ObjectOutputStream>::destroy<arrow::fs::(anonymous
namespace)::ObjectOutputStream> (__p=0xe2158005b40, this=0xe2158005b40) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/ext/new_allocator.h:168
#6 std::allocator_traits<std::allocator<arrow::fs::(anonymous
namespace)::ObjectOutputStream> >::destroy<arrow::fs::(anonymous
namespace)::ObjectOutputStream> (__p=0xe2158005b40, __a=...) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/alloc_traits.h:535
#7 std::_Sp_counted_ptr_inplace<arrow::fs::(anonymous
namespace)::ObjectOutputStream, std::allocator<arrow::fs::(anonymous
namespace)::ObjectOutputStream>, (__gnu_cxx::_Lock_policy)2>::_M_dispose
(this=0xe2158005b30) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:528
#8 0x00000e223c41ddda in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0xe2158005b30) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:168
#9 0x00000e223bbb62a8 in
std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=0xe21580f2b40, __in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:705
#10 std::__shared_ptr<arrow::io::OutputStream,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0xe21580f2b38,
__in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:1154
#11 std::shared_ptr<arrow::io::OutputStream>::~shared_ptr (this=0xe21580f2b38,
__in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr.h:122
#12 arrow::dataset::FileWriter::~FileWriter (this=0xe21580f2b10,
__in_chrg=<optimized out>) at
/tmp/build/ts/arrow/dataset/c/src/arrow/dataset/file_base.h:378
#13 arrow::dataset::ParquetFileWriter::~ParquetFileWriter (this=0xe21580f2b10,
__in_chrg=<optimized out>) at
/tmp/build/ts/arrow/dataset/c/src/arrow/dataset/file_parquet.h:282
#14 arrow::dataset::ParquetFileWriter::~ParquetFileWriter (this=0xe21580f2b10,
__in_chrg=<optimized out>) at
/tmp/build/ts/arrow/dataset/c/src/arrow/dataset/file_parquet.h:282
#15 0x00000e223c41ddda in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0xe21580b9a10) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:168
#16 0x00000e223bae3f1c in
std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=0x5b665b834690, __in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:705
#17 std::__shared_ptr<arrow::dataset::FileWriter,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x5b665b834688,
__in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:1154
#18 std::shared_ptr<arrow::dataset::FileWriter>::~shared_ptr
(this=0x5b665b834688, __in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr.h:122
#19 arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::~DatasetWriterFileQueue
(this=0x5b665b834670, __in_chrg=<optimized out>) at
/tmp/build/ts/arrow/dataset/c/src/arrow/dataset/dataset_writer.cc:140
#20 std::default_delete<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue>::operator() (__ptr=0x5b665b834670,
this=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:85
#21 std::unique_ptr<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue,
std::default_delete<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue> >::~unique_ptr (this=0x5b665b833800,
__in_chrg=<optimized out>) at
/tmp/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:361
#22 ~<lambda> (this=0x5b665b8337f8, __in_chrg=<optimized out>) at
/tmp/build/ts/arrow/dataset/c/src/arrow/dataset/dataset_writer.cc:364
#23
arrow::internal::FnOnce<arrow::Status()>::FnImpl<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterDirectoryQueue::OpenFileQueue(const
string&)::<lambda()> >::~FnImpl (this=0x5b665b8337f0, __in_chrg=<optimized
out>) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/functional.h:150
#24
arrow::internal::FnOnce<arrow::Status()>::FnImpl<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterDirectoryQueue::OpenFileQueue(const
string&)::<lambda()> >::~FnImpl(void) (this=0x5b665b8337f0,
__in_chrg=<optimized out>) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/functional.h:150
#25 0x00000e223995cd6e in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0x5b665b834970) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:168
#26 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=<optimized out>, __in_chrg=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:705
#27 std::__shared_ptr<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::State,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=<optimized out>,
__in_chrg=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr_base.h:1154
#28 std::shared_ptr<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::State>::~shared_ptr (this=<optimized out>,
__in_chrg=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/shared_ptr.h:122
#29 ~<lambda> (this=<optimized out>, __in_chrg=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/async_util.cc:428
#30
arrow::Future<arrow::internal::Empty>::ThenOnComplete<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::AddTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>)::WrapperTask::operator()()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::AddTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>)::WrapperTask::operator()()::<lambda()>
> >::operator() (result=..., this=0xe21800fa338) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.h:549
#31
arrow::Future<arrow::internal::Empty>::WrapResultOnComplete::Callback<arrow::Future<>::ThenOnComplete<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::AddTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>)::WrapperTask::operator()()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::AddTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>)::WrapperTask::operator()()::<lambda()>
> > >::operator() (impl=..., this=0xe21800fa338) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.h:442
#32 arrow::internal::FnOnce<void(const
arrow::FutureImpl&)>::FnImpl<arrow::Future<>::WrapResultOnComplete::Callback<arrow::Future<>::ThenOnComplete<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::AddTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>)::WrapperTask::operator()()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::util::(anonymous
namespace)::AsyncTaskGroupImpl::AddTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>)::WrapperTask::operator()()::<lambda()>
> > > >::invoke(const arrow::FutureImpl &) (this=0xe21800fa330, a#0=...) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/functional.h:152
#33 0x00000e22399a02a0 in arrow::internal::FnOnce<void (arrow::FutureImpl
const&)>::operator()(arrow::FutureImpl const&) && (a#0=..., this=0xe2180032070)
at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:173
#34 arrow::ConcreteFutureImpl::RunOrScheduleCallback
(self=std::shared_ptr<arrow::FutureImpl> (use count 2, weak count 1) = {...},
callback_record=..., in_add_callback=in_add_callback@entry=false) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:110
#35 0x00000e22399a05dd in arrow::ConcreteFutureImpl::DoMarkFinishedOrFailed
(this=<optimized out>, state=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:148
#36 0x00000e223999d83a in arrow::ConcreteFutureImpl::DoMarkFailed
(this=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:41
#37 0x00000e223bedc0a5 in arrow::Future<arrow::internal::Empty>::DoMarkFinished
(res=..., this=0xe2199eed210) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:660
#38 arrow::Future<arrow::internal::Empty>::MarkFinished<arrow::internal::Empty,
void> (this=0xe2199eed210, s=...) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:409
#39 0x00000e223bae3395 in
arrow::detail::ContinueFuture::operator()<arrow::Future<>::PassthruOnFailure<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()> >, const
arrow::Status&> (this=<synthetic pointer>, f=..., next=...) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:167
#40
arrow::Future<arrow::internal::Empty>::ThenOnComplete<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()> > >::operator()
(result=..., this=<optimized out>) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:550
#41
arrow::Future<arrow::internal::Empty>::WrapResultOnComplete::Callback<arrow::Future<>::ThenOnComplete<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()> > > >::operator()
(impl=..., this=<optimized out>) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:442
#42 arrow::internal::FnOnce<void(const
arrow::FutureImpl&)>::FnImpl<arrow::Future<>::WrapResultOnComplete::Callback<arrow::Future<>::ThenOnComplete<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::dataset::internal::(anonymous
namespace)::DatasetWriterFileQueue::DoFinish()::<lambda()> > > >
>::invoke(const arrow::FutureImpl &) (this=<optimized out>, a#0=...) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/functional.h:152
#43 0x00000e22399a02a0 in arrow::internal::FnOnce<void (arrow::FutureImpl
const&)>::operator()(arrow::FutureImpl const&) && (a#0=..., this=0xe21800fa0d0)
at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:173
#44 arrow::ConcreteFutureImpl::RunOrScheduleCallback
(self=std::shared_ptr<arrow::FutureImpl> (use count 2, weak count 1) = {...},
callback_record=..., in_add_callback=in_add_callback@entry=false) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:110
#45 0x00000e22399a05dd in arrow::ConcreteFutureImpl::DoMarkFinishedOrFailed
(this=<optimized out>, state=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:148
#46 0x00000e223999d83a in arrow::ConcreteFutureImpl::DoMarkFailed
(this=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:41
#47 0x00000e223bedc0a5 in arrow::Future<arrow::internal::Empty>::DoMarkFinished
(res=..., this=0xe218010a448) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:660
#48 arrow::Future<arrow::internal::Empty>::MarkFinished<arrow::internal::Empty,
void> (this=0xe218010a448, s=...) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:409
#49 0x00000e223bae8492 in
arrow::detail::MarkNextFinished<arrow::Future<arrow::internal::Empty>,
arrow::Future<arrow::internal::Empty>, true, true>::operator()(arrow::Status
const&) && (status=..., this=0xe218010a448) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:93
#50
arrow::Future<arrow::internal::Empty>::WrapStatusyOnComplete::Callback<arrow::detail::MarkNextFinished<arrow::Future<arrow::internal::Empty>,
arrow::Future<arrow::internal::Empty>, true, true>
>::operator()(arrow::FutureImpl const&) && (impl=..., this=0xe218010a448) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:455
#51 arrow::internal::FnOnce<void (arrow::FutureImpl
const&)>::FnImpl<arrow::Future<arrow::internal::Empty>::WrapStatusyOnComplete::Callback<arrow::detail::MarkNextFinished<arrow::Future<arrow::internal::Empty>,
arrow::Future<arrow::internal::Empty>, true, true> >
>::invoke(arrow::FutureImpl const&) (this=0xe218010a440, a#0=...) at
/tmp/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/functional.h:152
#52 0x00000e22399a02a0 in arrow::internal::FnOnce<void (arrow::FutureImpl
const&)>::operator()(arrow::FutureImpl const&) && (a#0=..., this=0xe218010b230)
at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:173
#53 arrow::ConcreteFutureImpl::RunOrScheduleCallback
(self=std::shared_ptr<arrow::FutureImpl> (use count 2, weak count 1) = {...},
callback_record=..., in_add_callback=in_add_callback@entry=false) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:110
#54 0x00000e22399a05dd in arrow::ConcreteFutureImpl::DoMarkFinishedOrFailed
(this=<optimized out>, state=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:148
#55 0x00000e223999d83a in arrow::ConcreteFutureImpl::DoMarkFailed
(this=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:41
#56 0x00000e223bedc0a5 in arrow::Future<arrow::internal::Empty>::DoMarkFinished
(res=..., this=0xe2199eed610) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:660
#57 arrow::Future<arrow::internal::Empty>::MarkFinished<arrow::internal::Empty,
void> (this=0xe2199eed610, s=...) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:409
#58 0x00000e223a4c7a61 in
arrow::detail::ContinueFuture::operator()<arrow::Future<>::PassthruOnFailure<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()> >, const
arrow::Status&> (this=<synthetic pointer>, f=..., next=...) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.h:167
#59 arrow::Future<arrow::internal::Empty>::ThenOnComplete<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()> > >::operator()
(result=..., this=0xe218010a038) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.h:550
#60
arrow::Future<arrow::internal::Empty>::WrapResultOnComplete::Callback<arrow::Future<>::ThenOnComplete<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()> > > >::operator()
(impl=..., this=0xe218010a038) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.h:442
#61 arrow::internal::FnOnce<void(const
arrow::FutureImpl&)>::FnImpl<arrow::Future<>::WrapResultOnComplete::Callback<arrow::Future<>::ThenOnComplete<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()>,
arrow::Future<>::PassthruOnFailure<arrow::fs::(anonymous
namespace)::ObjectOutputStream::CloseAsync()::<lambda()> > > > >::invoke(const
arrow::FutureImpl &) (this=0xe218010a030, a#0=...) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/functional.h:152
#62 0x00000e22399a02a0 in arrow::internal::FnOnce<void (arrow::FutureImpl
const&)>::operator()(arrow::FutureImpl const&) && (a#0=..., this=0xe218010a810)
at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:173
#63 arrow::ConcreteFutureImpl::RunOrScheduleCallback
(self=std::shared_ptr<arrow::FutureImpl> (use count 2, weak count 1) = {...},
callback_record=..., in_add_callback=in_add_callback@entry=false) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:110
#64 0x00000e22399a05dd in arrow::ConcreteFutureImpl::DoMarkFinishedOrFailed
(this=<optimized out>, state=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:148
#65 0x00000e223999d83a in arrow::ConcreteFutureImpl::DoMarkFailed
(this=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.cc:41
#66 0x00000e223bedc0a5 in arrow::Future<arrow::internal::Empty>::DoMarkFinished
(res=..., this=0xe2158016cb0) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:660
#67 arrow::Future<arrow::internal::Empty>::MarkFinished<arrow::internal::Empty,
void> (this=0xe2158016cb0, s=...) at
/build/build/ext/public/apache/arrow/15/0/0/dist/include/arrow/util/future.h:409
#68 0x00000e223a4ba9a3 in arrow::fs::(anonymous
namespace)::ObjectOutputStream::HandleUploadOutcome (result=..., req=...,
part_number=<optimized out>, state=...) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/filesystem/s3fs.cc:1687
#69 operator() (__closure=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/filesystem/s3fs.cc:1660
#70 0x00000e223a4babdc in
arrow::detail::ContinueFuture::operator()<arrow::fs::(anonymous
namespace)::ObjectOutputStream::UploadPart(void const*, int64_t,
std::shared_ptr<arrow::Buffer>)::<lambda()>&> (this=<optimized out>, f=...,
next=...) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/future.h:150
#71 std::__invoke_impl<void, arrow::detail::ContinueFuture&,
arrow::Future<arrow::internal::Empty>&, arrow::fs::(anonymous
namespace)::ObjectOutputStream::UploadPart(void const*, int64_t,
std::shared_ptr<arrow::Buffer>)::<lambda()>&> (__f=...) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/invoke.h:61
#72 std::__invoke<arrow::detail::ContinueFuture&,
arrow::Future<arrow::internal::Empty>&, arrow::fs::(anonymous
namespace)::ObjectOutputStream::UploadPart(void const*, int64_t,
std::shared_ptr<arrow::Buffer>)::<lambda()>&> (__fn=...) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/invoke.h:96
#73
std::_Bind<arrow::detail::ContinueFuture(arrow::Future<arrow::internal::Empty>,
arrow::fs::(anonymous namespace)::ObjectOutputStream::UploadPart(void const*,
int64_t, std::shared_ptr<arrow::Buffer>)::<lambda()>)>::__call<void, 0, 1>
(__args=..., this=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/functional:420
#74
std::_Bind<arrow::detail::ContinueFuture(arrow::Future<arrow::internal::Empty>,
arrow::fs::(anonymous namespace)::ObjectOutputStream::UploadPart(void const*,
int64_t, std::shared_ptr<arrow::Buffer>)::<lambda()>)>::operator()<>
(this=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/functional:503
#75
arrow::internal::FnOnce<void()>::FnImpl<std::_Bind<arrow::detail::ContinueFuture(arrow::Future<arrow::internal::Empty>,
arrow::fs::(anonymous namespace)::ObjectOutputStream::UploadPart(void const*,
int64_t, std::shared_ptr<arrow::Buffer>)::<lambda()>)> >::invoke(void)
(this=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/functional.h:152
#76 0x00000e22399cfbec in arrow::internal::FnOnce<void ()>::operator()() &&
(this=0xe2199eede80) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/unique_ptr.h:173
#77 arrow::internal::WorkerLoop (it={_M_id = {_M_thread = 15537479280384}},
state=...) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/thread_pool.cc:457
#78 operator() (__closure=<optimized out>) at
/build/build/ext/public/apache/arrow/15/0/0/apache-arrow/cpp/src/arrow/util/thread_pool.cc:618
#79 std::__invoke_impl<void,
arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()> > (__f=...)
at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/invoke.h:61
#80
std::__invoke<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> (__fn=...) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/invoke.h:96
#81
std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> >::_M_invoke<0> (this=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/std_thread.h:253
#82
std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> >::operator() (this=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/std_thread.h:260
#83
std::thread::_State_impl<std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> > >::_M_run(void) (this=<optimized out>) at
/build/build/ext/public/gpl3/gnu/gcc/11/dist/include/c++/11.3.0/bits/std_thread.h:211
#84 0x00000e2238ee74c4 in std::execute_native_thread_routine
(__p=0x5b665d38c7b0) at
/build/build/ext/public/gpl3/gnu/gcc/11/build/extract/gcc-11.3.0/libstdc++-v3/src/c++11/thread.cc:82
#85 0x00000e223fe11ea7 in start_thread (arg=<optimized out>) at
pthread_create.c:477
#86 0x00000e223ff27a6f in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:95
