Add bit test/set/clear/assign functions which prevents certain compiler optimizations and guarantees that program-level memory loads and/or stores will actually occur.
These functions are useful when interacting with memory-mapped hardware devices. The "once" family of functions does not promise atomicity and provides no memory ordering guarantees beyond the C11 relaxed memory model. RFC v3: * Work around lack of C++ support for _Generic (Tyler Retzlaff). Signed-off-by: Mattias Rönnblom <mattias.ronnb...@ericsson.com> --- lib/eal/include/rte_bitops.h | 180 +++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/lib/eal/include/rte_bitops.h b/lib/eal/include/rte_bitops.h index fb2e3dae7b..eac3f8b86a 100644 --- a/lib/eal/include/rte_bitops.h +++ b/lib/eal/include/rte_bitops.h @@ -201,6 +201,147 @@ extern "C" { uint32_t *: __rte_bit_assign32, \ uint64_t *: __rte_bit_assign64)(addr, nr, value) +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Generic selection macro to test exactly once the value of a bit in + * a 32-bit or 64-bit word. The type of operation depends on the type + * of the @c addr parameter. + * + * This function is guaranteed to result in exactly one memory load + * (e.g., it may not be eliminate or merged by the compiler). + * + * \code{.c} + * rte_bit_once_set(addr, 17); + * if (rte_bit_once_test(addr, 17)) { + * ... + * } + * \endcode + * + * In the above example, rte_bit_once_set() may not be removed by + * the compiler, which would be allowed in case rte_bit_set() and + * rte_bit_test() was used. + * + * \code{.c} + * while (rte_bit_once_test(addr, 17); + * ; + * \endcode + * + * In case rte_bit_test(addr, 17) was used instead, the resulting + * object code could (and in many cases would be) replaced with + * the equivalent to + * \code{.c} + * if (rte_bit_test(addr, 17)) { + * for (;;) // spin forever + * ; + * } + * \endcode + * + * rte_bit_once_test() does not give any guarantees in regards to + * memory ordering or atomicity. + * + * The regular bit set operations (e.g., rte_bit_test()) should be + * preferred over the "once" family of operations (e.g., + * rte_bit_once_test()) if possible, since the latter may prevent + * optimizations crucial for run-time performance. + * + * @param addr + * A pointer to the word to query. + * @param nr + * The index of the bit. + * @return + * Returns true if the bit is set, and false otherwise. + */ + +#define rte_bit_once_test(addr, nr) \ + _Generic((addr), \ + uint32_t *: __rte_bit_once_test32, \ + uint64_t *: __rte_bit_once_test64)(addr, nr) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Set bit in word exactly once. + * + * Set bit specified by @c nr in the word pointed to by @c addr to '1' + * exactly once. + * + * This function is guaranteed to result in exactly one memory load + * and exactly one memory store, *or* an atomic bit set operation. + * + * See rte_bit_test_once32() for more information and uses cases for + * the "once" class of functions. + * + * This macro does not give any guarantees in regards to memory + * ordering or atomicity. + * + * @param addr + * A pointer to the word to modify. + * @param nr + * The index of the bit. + */ +#define rte_bit_once_set(addr, nr) \ + _Generic((addr), \ + uint32_t *: __rte_bit_once_set32, \ + uint64_t *: __rte_bit_once_set64)(addr, nr) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Clear bit in word exactly once. + * + * Set bit specified by @c nr in the word pointed to by @c addr to '0' + * exactly once. + * + * This function is guaranteed to result in exactly one memory load + * and exactly one memory store, *or* an atomic bit clear operation. + * + * See rte_bit_test_once32() for more information and uses cases for + * the "once" class of functions. + * + * This macro does not give any guarantees in regards to memory + * ordering or atomicity. + * + * @param addr + * A pointer to the word to modify. + * @param nr + * The index of the bit. + */ +#define rte_bit_once_clear(addr, nr) \ + _Generic((addr), \ + uint32_t *: __rte_bit_once_clear32, \ + uint64_t *: __rte_bit_once_clear64)(addr, nr) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Assign a value to bit in a word exactly once. + * + * Set bit specified by @c nr in the word pointed to by @c addr to the + * value indicated by @c value exactly once. + * + * This function is guaranteed to result in exactly one memory load + * and exactly one memory store, *or* an atomic bit clear operation. + * + * This function does not give any guarantees in regards to memory + * ordering or atomicity. + * + * @param addr + * A pointer to the word to modify. + * @param nr + * The index of the bit. + * @param value + * The new value of the bit - true for '1', or false for '0'. + */ +#define rte_bit_once_assign(addr, nr, value) \ + _Generic((addr), \ + uint32_t *: __rte_bit_once_assign32, \ + uint64_t *: __rte_bit_once_assign64)(addr, nr, value) + #define __RTE_GEN_BIT_TEST(name, size, qualifier) \ static inline bool \ name(const qualifier uint ## size ## _t *addr, unsigned int nr) \ @@ -239,6 +380,14 @@ __RTE_GEN_BIT_TEST(__rte_bit_test64, 64, ) __RTE_GEN_BIT_SET(__rte_bit_set64, 64, ) __RTE_GEN_BIT_CLEAR(__rte_bit_clear64, 64, ) +__RTE_GEN_BIT_TEST(__rte_bit_once_test32, 32, volatile) +__RTE_GEN_BIT_SET(__rte_bit_once_set32, 32, volatile) +__RTE_GEN_BIT_CLEAR(__rte_bit_once_clear32, 32, volatile) + +__RTE_GEN_BIT_TEST(__rte_bit_once_test64, 64, volatile) +__RTE_GEN_BIT_SET(__rte_bit_once_set64, 64, volatile) +__RTE_GEN_BIT_CLEAR(__rte_bit_once_clear64, 64, volatile) + __rte_experimental static inline void __rte_bit_assign32(uint32_t *addr, unsigned int nr, bool value) @@ -259,6 +408,27 @@ __rte_bit_assign64(uint64_t *addr, unsigned int nr, bool value) __rte_bit_clear64(addr, nr); } + +__rte_experimental +static inline void +__rte_bit_once_assign32(uint32_t *addr, unsigned int nr, bool value) +{ + if (value) + __rte_bit_once_set32(addr, nr); + else + __rte_bit_once_clear32(addr, nr); +} + +__rte_experimental +static inline void +__rte_bit_once_assign64(volatile uint64_t *addr, unsigned int nr, bool value) +{ + if (value) + __rte_bit_once_set64(addr, nr); + else + __rte_bit_once_clear64(addr, nr); +} + /*------------------------ 32-bit relaxed operations ------------------------*/ /** @@ -953,6 +1123,11 @@ rte_log2_u64(uint64_t v) #undef rte_bit_clear #undef rte_bit_assign +#undef rte_bit_once_test +#undef rte_bit_once_set +#undef rte_bit_once_clear +#undef rte_bit_once_assign + #define __RTE_BIT_OVERLOAD_SZ_2(fun, qualifier, size, arg1_type, arg1_name) \ static inline void \ rte_bit_ ## fun(qualifier uint ## size ## _t *addr, \ @@ -1001,6 +1176,11 @@ __RTE_BIT_OVERLOAD_2(set, , unsigned int, nr) __RTE_BIT_OVERLOAD_2(clear, , unsigned int, nr) __RTE_BIT_OVERLOAD_3(assign, , unsigned int, nr, bool, value) +__RTE_BIT_OVERLOAD_2R(once_test, const volatile, bool, unsigned int, nr) +__RTE_BIT_OVERLOAD_2(once_set, volatile, unsigned int, nr) +__RTE_BIT_OVERLOAD_2(once_clear, volatile, unsigned int, nr) +__RTE_BIT_OVERLOAD_3(once_assign, volatile, unsigned int, nr, bool, value) + #endif #endif /* _RTE_BITOPS_H_ */ -- 2.34.1