Author: Hui Date: 2026-01-21T08:02:39Z New Revision: 3d4c82788af454c274a701a895a866f50ecb9b88
URL: https://github.com/llvm/llvm-project/commit/3d4c82788af454c274a701a895a866f50ecb9b88 DIFF: https://github.com/llvm/llvm-project/commit/3d4c82788af454c274a701a895a866f50ecb9b88.diff LOG: [libc++] Introduce a native timed wait in the synchronization library (#172214) Fixes #172137 (cherry picked from commit 2e53764f2da742ba32b333e33635af60d384c2a8) Added: libcxx/include/__atomic/atomic_sync_timed.h libcxx/test/std/thread/thread.semaphore/lost_wakeup.timed.pass.cpp Modified: libcxx/include/CMakeLists.txt libcxx/include/atomic libcxx/include/module.modulemap.in libcxx/include/semaphore libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist libcxx/src/atomic.cpp Removed: ################################################################################ diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index f9ae22accd687..787cfffa50195 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -214,6 +214,7 @@ set(files __atomic/atomic_lock_free.h __atomic/atomic_ref.h __atomic/atomic_sync.h + __atomic/atomic_sync_timed.h __atomic/atomic_waitable_traits.h __atomic/check_memory_order.h __atomic/contention_t.h diff --git a/libcxx/include/__atomic/atomic_sync_timed.h b/libcxx/include/__atomic/atomic_sync_timed.h new file mode 100644 index 0000000000000..7daff73db7ebd --- /dev/null +++ b/libcxx/include/__atomic/atomic_sync_timed.h @@ -0,0 +1,144 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H +#define _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H + +#include <__atomic/atomic_waitable_traits.h> +#include <__atomic/contention_t.h> +#include <__atomic/memory_order.h> +#include <__atomic/to_gcc_order.h> +#include <__chrono/duration.h> +#include <__config> +#include <__memory/addressof.h> +#include <__thread/poll_with_backoff.h> +#include <__thread/timed_backoff_policy.h> +#include <__type_traits/conjunction.h> +#include <__type_traits/decay.h> +#include <__type_traits/has_unique_object_representation.h> +#include <__type_traits/invoke.h> +#include <__type_traits/is_same.h> +#include <__type_traits/is_trivially_copyable.h> +#include <__type_traits/void_t.h> +#include <__utility/declval.h> +#include <cstdint> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 +# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +_LIBCPP_AVAILABILITY_NEW_SYNC +_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __address) _NOEXCEPT; + +// wait on the global contention state to be changed from the given value for the address +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout( + void const* __address, __cxx_contention_t __monitor_value, uint64_t __timeout_ns) _NOEXCEPT; + +// wait on the address directly with the native platform wait +template <std::size_t _Size> +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) _NOEXCEPT; + +template <class _AtomicWaitable, class _Poll, class _Rep, class _Period> +struct __atomic_wait_timed_backoff_impl { + const _AtomicWaitable& __a_; + _Poll __poll_; + memory_order __order_; + chrono::duration<_Rep, _Period> __rel_time_; + + using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type; + + _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const { + if (__elapsed > chrono::microseconds(4)) { + auto __contention_address = const_cast<const void*>( + static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_))); + + uint64_t __timeout_ns = + static_cast<uint64_t>((chrono::duration_cast<chrono::nanoseconds>(__rel_time_) - __elapsed).count()); + + if constexpr (__has_native_atomic_wait<__value_type>) { + auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_); + if (__poll_(__atomic_value)) + return __backoff_results::__poll_success; + std::__atomic_wait_native_with_timeout<sizeof(__value_type)>( + __contention_address, std::addressof(__atomic_value), __timeout_ns); + } else { + __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address); + auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_); + if (__poll_(__atomic_value)) + return __backoff_results::__poll_success; + std::__atomic_wait_global_table_with_timeout(__contention_address, __monitor_val, __timeout_ns); + } + } else { + } // poll + return __backoff_results::__continue_poll; + } +}; + +// The semantics of this function are similar to `atomic`'s +// `.wait(T old, std::memory_order order)` with a timeout, but instead of having a hardcoded +// predicate (is the loaded value unequal to `old`?), the predicate function is +// specified as an argument. The loaded value is given as an in-out argument to +// the predicate. If the predicate function returns `true`, +// `__atomic_wait_unless_with_timeout` will return. If the predicate function returns +// `false`, it must set the argument to its current understanding of the atomic +// value. The predicate function must not return `false` spuriously. +template <class _AtomicWaitable, class _Poll, class _Rep, class _Period> +_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout( + const _AtomicWaitable& __a, + memory_order __order, + _Poll&& __poll, + chrono::duration<_Rep, _Period> const& __rel_time) { + static_assert(__atomic_waitable<_AtomicWaitable>, ""); + __atomic_wait_timed_backoff_impl<_AtomicWaitable, __decay_t<_Poll>, _Rep, _Period> __backoff_fn = { + __a, __poll, __order, __rel_time}; + auto __poll_result = std::__libcpp_thread_poll_with_backoff( + /* poll */ + [&]() { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order); + return __poll(__current_val); + }, + /* backoff */ __backoff_fn, + __rel_time); + + return __poll_result == __poll_with_backoff_results::__poll_success; +} + +# elif _LIBCPP_HAS_THREADS // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +template <class _AtomicWaitable, class _Poll, class _Rep, class _Period> +_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout( + const _AtomicWaitable& __a, + memory_order __order, + _Poll&& __poll, + chrono::duration<_Rep, _Period> const& __rel_time) { + auto __res = std::__libcpp_thread_poll_with_backoff( + /* poll */ + [&]() { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order); + return __poll(__current_val); + }, + /* backoff */ __libcpp_timed_backoff_policy(), + __rel_time); + return __res == __poll_with_backoff_results::__poll_success; +} + +# endif // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H diff --git a/libcxx/include/atomic b/libcxx/include/atomic index 3323650f7f81a..23a3db5d35029 100644 --- a/libcxx/include/atomic +++ b/libcxx/include/atomic @@ -608,6 +608,7 @@ template <class T> # include <__atomic/atomic_init.h> # include <__atomic/atomic_lock_free.h> # include <__atomic/atomic_sync.h> +# include <__atomic/atomic_sync_timed.h> # include <__atomic/atomic_waitable_traits.h> # include <__atomic/check_memory_order.h> # include <__atomic/contention_t.h> diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index 27178aa1f4378..6bee3809d7da5 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -869,6 +869,7 @@ module std [system] { module atomic_lock_free { header "__atomic/atomic_lock_free.h" } module atomic_ref { header "__atomic/atomic_ref.h" } module atomic_sync { header "__atomic/atomic_sync.h" } + module atomic_sync_timed { header "__atomic/atomic_sync_timed.h" } module atomic_waitable_traits { header "__atomic/atomic_waitable_traits.h" } module atomic { header "__atomic/atomic.h" diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index 7c1637356bfe5..d411d205cdf2d 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -55,12 +55,11 @@ using binary_semaphore = counting_semaphore<1>; // since C++20 # include <__assert> # include <__atomic/atomic.h> # include <__atomic/atomic_sync.h> +# include <__atomic/atomic_sync_timed.h> # include <__atomic/memory_order.h> # include <__chrono/time_point.h> # include <__cstddef/ptr diff _t.h> -# include <__thread/poll_with_backoff.h> # include <__thread/support.h> -# include <__thread/timed_backoff_policy.h> # include <limits> # include <version> @@ -107,9 +106,9 @@ public: _LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) { if (__rel_time == chrono::duration<_Rep, _Period>::zero()) return try_acquire(); - auto const __poll_fn = [this]() { return try_acquire(); }; - return std::__libcpp_thread_poll_with_backoff(__poll_fn, __libcpp_timed_backoff_policy(), __rel_time) == - __poll_with_backoff_results::__poll_success; + + return std::__atomic_wait_unless_with_timeout( + __a_, memory_order_relaxed, [this](ptr diff _t& __old) { return __try_acquire_impl(__old); }, __rel_time); } _LIBCPP_HIDE_FROM_ABI bool try_acquire() { auto __old = __a_.load(memory_order_relaxed); diff --git a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist index 9b31c5d13eab9..da1e340b1d4aa 100644 --- a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -1587,7 +1587,9 @@ {'is_defined': True, 'name': '__ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '__ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '__ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'} +{'is_defined': True, 'name': '__ZNSt3__133__atomic_wait_native_with_timeoutILm8EEEvPKvS2_y', 'type': 'FUNC'} {'is_defined': True, 'name': '__ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'} +{'is_defined': True, 'name': '__ZNSt3__139__atomic_wait_global_table_with_timeoutEPKvxy', 'type': 'FUNC'} {'is_defined': True, 'name': '__ZNSt3__13cinE', 'size': 0, 'type': 'OBJECT'} {'is_defined': True, 'name': '__ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '__ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'} @@ -2650,4 +2652,4 @@ {'is_defined': True, 'name': '___cxa_vec_new2', 'type': 'I'} {'is_defined': True, 'name': '___cxa_vec_new3', 'type': 'I'} {'is_defined': True, 'name': '___dynamic_cast', 'type': 'I'} -{'is_defined': True, 'name': '___gxx_personality_v0', 'type': 'I'} \ No newline at end of file +{'is_defined': True, 'name': '___gxx_personality_v0', 'type': 'I'} diff --git a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist index 3e16e595874fe..abbcc5c14b544 100644 --- a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -1223,7 +1223,9 @@ {'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt6__ndk133__atomic_wait_native_with_timeoutILj4EEEvPKvS2_y', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk134__construct_barrier_algorithm_baseERi', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt6__ndk139__atomic_wait_global_table_with_timeoutEPKviy', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 148, 'type': 'OBJECT'} {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD1Ev', 'type': 'FUNC'} diff --git a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist index b7987c0cdc1eb..1224f51e17687 100644 --- a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -571,6 +571,7 @@ {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'storage_mapping_class': 'DS', 'type': 'FUNC'} +{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__139__atomic_wait_global_table_with_timeoutEPKviy', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13cinE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'} @@ -1713,6 +1714,7 @@ {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm4EEEvPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'} +{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__133__atomic_wait_native_with_timeoutILm4EEEvPKvS2_y', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem16_FilesystemClock9is_steadyE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem4path19preferred_separatorE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__16__sortIRNS_6__lessIaaEEPaEEvT0_S5_T_', 'storage_mapping_class': 'DS', 'type': 'FUNC'} diff --git a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist index bdbee9a3e7089..934505b18a64b 100644 --- a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -571,6 +571,7 @@ {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'storage_mapping_class': 'DS', 'type': 'FUNC'} +{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__139__atomic_wait_global_table_with_timeoutEPKvlm', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13cinE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'} @@ -1713,6 +1714,7 @@ {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__126__atomic_notify_one_nativeILm8EEEvPKv', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIdEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__127__from_chars_floating_pointIfEENS_19__from_chars_resultIT_EEPKcS5_NS_12chars_formatE', 'storage_mapping_class': 'DS', 'type': 'FUNC'} +{'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__133__atomic_wait_native_with_timeoutILm8EEEvPKvS2_m', 'storage_mapping_class': 'DS', 'type': 'FUNC'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem16_FilesystemClock9is_steadyE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__14__fs10filesystem4path19preferred_separatorE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'} {'import_export': 'wEXP', 'is_defined': True, 'name': '_ZNSt3__16__sortIRNS_6__lessIaaEEPaEEvT0_S5_T_', 'storage_mapping_class': 'DS', 'type': 'FUNC'} diff --git a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist index d39dc09984ab7..6c437f9adea48 100644 --- a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -1223,7 +1223,9 @@ {'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt6__ndk133__atomic_wait_native_with_timeoutILm4EEEvPKvS2_m', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt6__ndk139__atomic_wait_global_table_with_timeoutEPKvim', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk13cinE', 'size': 280, 'type': 'OBJECT'} {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt6__ndk13pmr15memory_resourceD1Ev', 'type': 'FUNC'} diff --git a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist index e17e111cb5644..017b6d33c4262 100644 --- a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -1237,7 +1237,9 @@ {'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt3__133__atomic_wait_native_with_timeoutILm8EEEvPKvS2_m', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt3__139__atomic_wait_global_table_with_timeoutEPKvlm', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 400, 'type': 'OBJECT'} {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'} @@ -2026,4 +2028,4 @@ {'is_defined': True, 'name': '_ZTv0_n24_NSt3__114basic_iostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZTv0_n24_NSt3__114basic_iostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZTv0_n24_NSt3__19strstreamD0Ev', 'type': 'FUNC'} -{'is_defined': True, 'name': '_ZTv0_n24_NSt3__19strstreamD1Ev', 'type': 'FUNC'} \ No newline at end of file +{'is_defined': True, 'name': '_ZTv0_n24_NSt3__19strstreamD1Ev', 'type': 'FUNC'} diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist index e9f1cc5172ee5..363fa42975b8e 100644 --- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist +++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist @@ -1235,7 +1235,9 @@ {'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt3__133__atomic_wait_native_with_timeoutILm4EEEvPKvS2_m', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt3__139__atomic_wait_global_table_with_timeoutEPKvim', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'} {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'} @@ -2025,4 +2027,4 @@ {'is_defined': True, 'name': '_ZTv0_n24_NSt3__114basic_iostreamIcNS_11char_traitsIcEEED0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZTv0_n24_NSt3__114basic_iostreamIcNS_11char_traitsIcEEED1Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZTv0_n24_NSt3__19strstreamD0Ev', 'type': 'FUNC'} -{'is_defined': True, 'name': '_ZTv0_n24_NSt3__19strstreamD1Ev', 'type': 'FUNC'} \ No newline at end of file +{'is_defined': True, 'name': '_ZTv0_n24_NSt3__19strstreamD1Ev', 'type': 'FUNC'} diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist index 97a99f0601710..cf4668a2dea21 100644 --- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist +++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist @@ -1206,7 +1206,9 @@ {'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_all_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__132__atomic_notify_one_global_tableEPKv', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__132__destroy_barrier_algorithm_baseEPNS_24__barrier_algorithm_baseE', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt3__133__atomic_wait_native_with_timeoutILm4EEEvPKvS2_m', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__134__construct_barrier_algorithm_baseERl', 'type': 'FUNC'} +{'is_defined': True, 'name': '_ZNSt3__139__atomic_wait_global_table_with_timeoutEPKvim', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__13cinE', 'size': 280, 'type': 'OBJECT'} {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD0Ev', 'type': 'FUNC'} {'is_defined': True, 'name': '_ZNSt3__13pmr15memory_resourceD1Ev', 'type': 'FUNC'} diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp index 33ca5410923b6..637c971ed8896 100644 --- a/libcxx/src/atomic.cpp +++ b/libcxx/src/atomic.cpp @@ -10,6 +10,7 @@ #include <atomic> #include <climits> #include <cstddef> +#include <cstdint> #include <cstring> #include <functional> #include <new> @@ -63,11 +64,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD #ifdef __linux__ template <std::size_t _Size> -static void __platform_wait_on_address(void const* __ptr, void const* __val) { +static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) { static_assert(_Size == 4, "Can only wait on 4 bytes value"); char buffer[_Size]; std::memcpy(&buffer, const_cast<const void*>(__val), _Size); - static constexpr timespec __timeout = {2, 0}; + static constexpr timespec __default_timeout = {2, 0}; + timespec __timeout; + if (__timeout_ns == 0) { + __timeout = __default_timeout; + } else { + __timeout.tv_sec = __timeout_ns / 1'000'000'000; + __timeout.tv_nsec = __timeout_ns % 1'000'000'000; + } _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, *reinterpret_cast<__cxx_contention_t const*>(&buffer), &__timeout, 0, 0); } @@ -89,14 +97,17 @@ extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value) # define ULF_WAKE_ALL 0x00000100 template <std::size_t _Size> -static void __platform_wait_on_address(void const* __ptr, void const* __val) { +static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) { static_assert(_Size == 8 || _Size == 4, "Can only wait on 8 bytes or 4 bytes value"); char buffer[_Size]; std::memcpy(&buffer, const_cast<const void*>(__val), _Size); + auto __timeout_us = __timeout_ns == 0 ? 0 : static_cast<uint32_t>(__timeout_ns / 1000); if constexpr (_Size == 4) - __ulock_wait(UL_COMPARE_AND_WAIT, const_cast<void*>(__ptr), *reinterpret_cast<uint32_t const*>(&buffer), 0); + __ulock_wait( + UL_COMPARE_AND_WAIT, const_cast<void*>(__ptr), *reinterpret_cast<uint32_t const*>(&buffer), __timeout_us); else - __ulock_wait(UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), *reinterpret_cast<uint64_t const*>(&buffer), 0); + __ulock_wait( + UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), *reinterpret_cast<uint64_t const*>(&buffer), __timeout_us); } template <std::size_t _Size> @@ -117,11 +128,25 @@ static void __platform_wake_by_address(void const* __ptr, bool __notify_one) { */ template <std::size_t _Size> -static void __platform_wait_on_address(void const* __ptr, void const* __val) { +static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) { static_assert(_Size == 8, "Can only wait on 8 bytes value"); char buffer[_Size]; std::memcpy(&buffer, const_cast<const void*>(__val), _Size); - _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, *reinterpret_cast<__cxx_contention_t*>(&buffer), nullptr, nullptr); + if (__timeout_ns == 0) { + _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, *reinterpret_cast<__cxx_contention_t*>(&buffer), nullptr, nullptr); + } else { + _umtx_time ut; + ut._timeout.tv_sec = __timeout_ns / 1'000'000'000; + ut._timeout.tv_nsec = __timeout_ns % 1'000'000'000; + ut._flags = 0; // Relative time (not absolute) + ut._clockid = CLOCK_MONOTONIC; // Use monotonic clock + + _umtx_op(const_cast<void*>(__ptr), + UMTX_OP_WAIT, + *reinterpret_cast<__cxx_contention_t*>(&buffer), + reinterpret_cast<void*>(sizeof(ut)), // Pass size as uaddr + &ut); // Pass _umtx_time structure as uaddr2 + } } template <std::size_t _Size> @@ -157,17 +182,21 @@ static void* win32_get_synch_api_function(const char* function_name) { } template <std::size_t _Size> -static void __platform_wait_on_address(void const* __ptr, void const* __val) { +static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) { static_assert(_Size == 8, "Can only wait on 8 bytes value"); // WaitOnAddress was added in Windows 8 (build 9200) static auto wait_on_address = reinterpret_cast<BOOL(WINAPI*)(void*, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress")); if (wait_on_address != nullptr) { - wait_on_address(const_cast<void*>(__ptr), const_cast<void*>(__val), _Size, INFINITE); + wait_on_address(const_cast<void*>(__ptr), + const_cast<void*>(__val), + _Size, + __timeout_ns == 0 ? INFINITE : static_cast<DWORD>(__timeout_ns / 1'000'000)); } else { __libcpp_thread_poll_with_backoff( [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; }, - __libcpp_timed_backoff_policy()); + __libcpp_timed_backoff_policy(), + std::chrono::nanoseconds(__timeout_ns)); } } @@ -202,10 +231,11 @@ static void __platform_wake_by_address(void const* __ptr, bool __notify_one) { // Baseline is just a timed backoff template <std::size_t _Size> -static void __platform_wait_on_address(void const* __ptr, void const* __val) { +static void __platform_wait_on_address(void const* __ptr, void const* __val, uint64_t __timeout_ns) { __libcpp_thread_poll_with_backoff( [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; }, - __libcpp_timed_backoff_policy()); + __libcpp_timed_backoff_policy(), + std::chrono::nanoseconds(__timeout_ns)); } template <std::size_t _Size> @@ -229,14 +259,16 @@ __contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __add } template <std::size_t _Size> -static void -__contention_wait(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_wait, void const* __old_value) { +static void __contention_wait(__cxx_atomic_contention_t* __waiter_count, + void const* __address_to_wait, + void const* __old_value, + uint64_t __timeout_ns) { __cxx_atomic_fetch_add(__waiter_count, __cxx_contention_t(1), memory_order_relaxed); // https://llvm.org/PR109290 // There are no platform guarantees of a memory barrier in the platform wait implementation __cxx_atomic_thread_fence(memory_order_seq_cst); // We sleep as long as the monitored value hasn't changed. - __platform_wait_on_address<_Size>(__address_to_wait, __old_value); + __platform_wait_on_address<_Size>(__address_to_wait, __old_value, __timeout_ns); __cxx_atomic_fetch_sub(__waiter_count, __cxx_contention_t(1), memory_order_release); } @@ -299,7 +331,14 @@ _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept { auto const __entry = __get_global_contention_state(__location); __contention_wait<sizeof(__cxx_atomic_contention_t)>( - &__entry->__waiter_count, &__entry->__platform_state, &__old_value); + &__entry->__waiter_count, &__entry->__platform_state, &__old_value, 0); +} + +_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout( + void const* __location, __cxx_contention_t __old_value, uint64_t __timeout_ns) _NOEXCEPT { + auto const __entry = __get_global_contention_state(__location); + __contention_wait<sizeof(__cxx_atomic_contention_t)>( + &__entry->__waiter_count, &__entry->__platform_state, &__old_value, __timeout_ns); } _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept { @@ -314,7 +353,13 @@ _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __lo template <std::size_t _Size> _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept { - __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value); + __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value, 0); +} + +template <std::size_t _Size> +_LIBCPP_EXPORTED_FROM_ABI void +__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) noexcept { + __contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value, __timeout_ns); } template <std::size_t _Size> @@ -335,6 +380,8 @@ _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location # define _INSTANTIATE(_SIZE) \ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const*, void const*) noexcept; \ + template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<_SIZE>( \ + void const*, void const*, uint64_t) noexcept; \ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \ template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept; @@ -347,6 +394,9 @@ _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE) template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept; +template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<sizeof(__cxx_contention_t)>( + void const* __address, void const* __old_value, uint64_t) noexcept; + template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept; @@ -378,7 +428,7 @@ _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept { auto const __entry = __get_global_contention_state(const_cast<void const*>(__location)); __contention_wait<sizeof(__cxx_atomic_contention_t)>( - &__entry->__waiter_count, &__entry->__platform_state, &__old_value); + &__entry->__waiter_count, &__entry->__platform_state, &__old_value, 0); } _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept { @@ -397,7 +447,7 @@ _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept { auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location)); __contention_wait<sizeof(__cxx_atomic_contention_t)>( - __get_native_waiter_count(__location_cast), __location_cast, &__old_value); + __get_native_waiter_count(__location_cast), __location_cast, &__old_value, 0); } // this function is even unused in the old ABI diff --git a/libcxx/test/std/thread/thread.semaphore/lost_wakeup.timed.pass.cpp b/libcxx/test/std/thread/thread.semaphore/lost_wakeup.timed.pass.cpp new file mode 100644 index 0000000000000..aa18b8fc61a39 --- /dev/null +++ b/libcxx/test/std/thread/thread.semaphore/lost_wakeup.timed.pass.cpp @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: no-threads +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +// <semaphore> + +// Test that counting_semaphore::try_acquire_for does not suffer from lost wakeup +// under stress testing. + +#include <barrier> +#include <chrono> +#include <semaphore> +#include <thread> +#include <vector> + +#include "make_test_thread.h" + +static std::counting_semaphore<> s(0); +constexpr auto num_acquirer = 100; +constexpr auto num_iterations = 5000; +static std::barrier<> b(num_acquirer + 1); + +void acquire() { + for (int i = 0; i < num_iterations; ++i) { + while (!s.try_acquire_for(std::chrono::seconds(1))) { + } + } +} + +void release() { + for (int i = 0; i < num_iterations; ++i) { + s.release(num_acquirer); + } +} + +int main(int, char**) { + std::vector<std::thread> threads; + for (int i = 0; i < num_acquirer; ++i) + threads.push_back(support::make_test_thread(acquire)); + + threads.push_back(support::make_test_thread(release)); + + for (auto& thread : threads) + thread.join(); + + return 0; +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
