From: Geoffrey Blake <geoffrey.bl...@arm.com>
Create an opaque tagged pointer type for 32 and 64bit
architectures with large tags to avoid the ABA problem with the lockless stack
used in the buffer allocation code. Compiles cleanly under
x86_64 and i386 and passes tests that stress allocator.
Signed-off-by: Geoffrey Blake <geoffrey.bl...@arm.com>
---
(This document/code contribution attached is provided under the terms of
agreement LES-LTM-21309)
configure.ac | 13 ++
.../linux-generic/include/odp_atomic_internal.h | 242 ++++++++++++++++++++-
2 files changed, 253 insertions(+), 2 deletions(-)
diff --git a/configure.ac b/configure.ac
index e5c1c56..3ba0a8c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -184,6 +184,7 @@ AC_HELP_STRING([--with-openssl-path=DIR path to openssl
libs and headers],
##########################################################################
OLD_LDFLAGS=$LDFLAGS
OLD_CPPFLAGS=$CPPFLAGS
+OLD_CFLAGS=$CFLAGS
LDFLAGS="$AM_LDFLAGS $LDFLAGS"
CPPFLAGS="$AM_CPPFLAGS $CPPFLAGS"
@@ -211,10 +212,22 @@ AC_CHECK_HEADERS([openssl/des.h openssl/rand.h
openssl/hmac.h openssl/evp.h], []
[AC_MSG_ERROR([OpenSSL headers required])])
##########################################################################
+# Check if compiler supports cmpxchng16
+##########################################################################
+CFLAGS=-mcx16
+AC_MSG_CHECKING([whether CC supports -mcx16])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])],
+ [AC_MSG_RESULT([yes])]
+ [ODP_CFLAGS="$ODP_CFLAGS -mcx16"],
+ [AC_MSG_RESULT([no])]
+)
+
+##########################################################################
# Restore old saved variables
##########################################################################
LDFLAGS=$OLD_LDFLAGS
CPPFLAGS=$OLD_CPPFLAGS
+CFLAGS=$OLD_CFLAGS
##########################################################################
# Default warning setup
diff --git a/platform/linux-generic/include/odp_atomic_internal.h
b/platform/linux-generic/include/odp_atomic_internal.h
index a02ecc5..3ac9720 100644
--- a/platform/linux-generic/include/odp_atomic_internal.h
+++ b/platform/linux-generic/include/odp_atomic_internal.h
@@ -36,7 +36,33 @@ extern "C" {
typedef struct {
void *v; /**< Actual storage for the atomic variable */
} _odp_atomic_ptr_t
-ODP_ALIGNED(sizeof(void *)); /* Enforce alignement! */
+ODP_ALIGNED(sizeof(void *)); /* Enforce alignment! */
+
+/**
+ * Opaque tagged pointer atomic type
+ */
+#if defined __SIZEOF_INT128 && defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
+#define ODP_64BIT_HAVE_CMPX16
+/* Our system is 64bit and has cmpx16 support */
+typedef union {
+ __uint128_t v;
+ struct {
+ uint64_t tag;
+ void *ptr;
+ } tptr;
+} _odp_atomic_tptr_t
+ODP_ALIGNED(sizeof(__int128));
+#else
+/* Fallback path for all other archs */
+typedef struct {
+ char lock;
+ struct {
+ uint64_t tag;
+ void *ptr;
+ } tptr;
+} _odp_atomic_tptr_t
+ODP_ALIGNED(16);
+#endif
/**
* Atomic flag (boolean) type
@@ -471,7 +497,7 @@ static inline void _odp_atomic_ptr_init(_odp_atomic_ptr_t
*atom, void *val)
*
* @return Value of the variable
*/
-static inline void *_odp_atomic_ptr_load(const _odp_atomic_ptr_t *atom,
+static inline void *_odp_atomic_ptr_load(_odp_atomic_ptr_t *atom,
_odp_memmodel_t mmodel)
{
return __atomic_load_n(&atom->v, mmodel);
@@ -534,6 +560,218 @@ static inline int _odp_atomic_ptr_cmp_xchg_strong(
}
/*****************************************************************************
+ * Operations on tagged pointer atomics
+ * _odp_atomic_tptr_init - no return value
+ * _odp_atomic_tptr_load - return current value
+ * _odp_atomic_tptr_store - no return value
+ * _odp_atomic_tptr_xchg - return old value
+ * _odp_atomic_tptr_gettag - return tag value
+ * _odp_atomic_tptr_settag - no return value
+ * _odp_atomic_tptr_getptr - return pointer value
+ * _odp_atomic_tptr_setptr - no return value
+ *****************************************************************************/
+#if !defined ODP_64BIT_HAVE_CMPX16
+/**
+ * @internal
+ * Helper macro for lock-based atomic operations on 128-bit tagged pointers
+ * @param[in,out] atom Pointer to the atomic tagged pointer variable
+ * @param expr Expression used update the variable.
+ * @param mm Memory order to use.
+ * @return The old value of the variable.
+ */
+#define ATOMIC_TPTR_OP_MM(atom, expr, mm) \
+({ \
+ _odp_atomic_tptr_t old_val; \
+ /* Loop while lock is already taken, stop when lock becomes clear */ \
+ while (__atomic_test_and_set(&(atom)->lock, \
+ (mm) == _ODP_MEMMODEL_SC ? \
+ __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) \
+ (void)0; \
+ old_val.tptr = (atom->tptr); \
+ (expr); /* Perform whatever update is desired */ \
+ __atomic_clear(&(atom)->lock, \
+ (mm) == _ODP_MEMMODEL_SC ? \
+ __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); \
+ __atomic_clear(&(old_val).lock, \
+ (mm) == _ODP_MEMMODEL_SC ? \
+ __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); \
+ old_val; /* Return old value */ \
+})
+#endif
+
+/**
+ * Initialization of a tagged pointer atomic variable
+ *
+ * @param[out] atom Pointer to a pointer atomic variable
+ * @param val Value to initialize the variable with
+ */
+static inline void _odp_atomic_tptr_init(_odp_atomic_tptr_t *atom, void *val)
+{
+ __atomic_store_n(&atom->tptr.ptr, val, __ATOMIC_RELAXED);
+ __atomic_store_n(&atom->tptr.tag, 0, __ATOMIC_RELAXED);
+#if !defined ODP_64BIT_HAVE_CMPX16
+ __atomic_clear(&atom->lock, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * Atomic load of a tagged pointer atomic variable
+ *
+ * @param atom Tagged pointer to a pointer atomic variable
+ * @param mmodel Memory order associated with the load operation
+ *
+ * @return Value of the variable
+ */
+static inline _odp_atomic_tptr_t _odp_atomic_tptr_load(_odp_atomic_tptr_t
*atom,
+ _odp_memmodel_t mmodel)
+{
+#if defined ODP_64BIT_HAVE_CMPX16
+ _odp_atomic_tptr_t ret;
+ ret.v = __atomic_load_n(&atom->v, mmodel);
+ return ret;
+#else
+ return ATOMIC_TPTR_OP_MM(atom, (void)0, mmodel);
+#endif
+}
+
+/**
+ * Atomic store to a tagged pointer atomic variable
+ *
+ * @param[out] atom Tagged pointer to a pointer atomic variable
+ * @param val Value to write to the atomic variable
+ * @param mmodel Memory order associated with the store operation
+ */
+static inline void _odp_atomic_tptr_store(_odp_atomic_tptr_t *atom,
+ _odp_atomic_tptr_t *val,
+ _odp_memmodel_t mmodel)
+{
+#if defined ODP_64BIT_HAVE_CMPX16
+ __atomic_store_n(&atom->v, val->v, mmodel);
+#else
+ ATOMIC_TPTR_OP_MM(atom, atom->tptr = val->tptr, mmodel);
+#endif
+}
+
+/**
+ * Atomic exchange (swap) of tagged pointer atomic variable
+ *
+ * @param[in,out] atom Tagged pointer to a pointer atomic variable
+ * @param val New value to write
+ * @param mmodel Memory order associated with the exchange operation
+ *
+ * @return Old value of variable
+ */
+static inline _odp_atomic_tptr_t _odp_atomic_tptr_xchg(_odp_atomic_tptr_t
*atom,
+ _odp_atomic_tptr_t *val,
+ _odp_memmodel_t mmodel)
+{
+#if defined ODP_64BIT_HAVE_CMPX16
+ _odp_atomic_tptr_t ret;
+ ret.v = __atomic_exchange_n(&atom->v, val->v, mmodel);
+ return ret;
+#else
+ return ATOMIC_TPTR_OP_MM(atom, atom->tptr = val->tptr, mmodel);
+#endif
+}
+
+/**
+ * Atomic compare and exchange (swap) of pointer atomic variable
+ * "Strong" semantics, will not fail spuriously.
+ *
+ * @param[in,out] atom Pointer to a pointer atomic variable
+ * @param[in,out] exp Pointer to expected value (updated on failure)
+ * @param val New value to write
+ * @param success Memory order associated with a successful compare-and-swap
+ * operation
+ * @param failure Memory order associated with a failed compare-and-swap
+ * operation
+ *
+ * @retval 1 exchange successful
+ * @retval 0 exchange failed and '*exp' updated with current value
+ */
+static inline int _odp_atomic_tptr_cmp_xchg_strong(
+ _odp_atomic_tptr_t *atom,
+ _odp_atomic_tptr_t *exp,
+ _odp_atomic_tptr_t *val,
+ _odp_memmodel_t success,
+ _odp_memmodel_t failure)
+{
+#if defined ODP_64BIT_HAVE_CMPX16
+ return __atomic_compare_exchange_n(&atom->v, &exp->v, val->v,
+ false/*strong*/, success, failure);
+#else
+ /* Possibly we are a bit pessimistic with the memory models */
+ odp_bool_t ret_succ;
+ /* Loop while lock is already taken, stop when lock becomes clear */
+ while (__atomic_test_and_set(&(atom)->lock,
+ (success) == _ODP_MEMMODEL_SC ?
+ __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE))
+ (void)0;
+ if (atom->tptr.tag == exp->tptr.tag &&
+ atom->tptr.ptr == exp->tptr.ptr) {
+ atom->tptr = val->tptr;
+ ret_succ = 1;
+ } else {
+ exp->tptr = atom->tptr;
+ ret_succ = 0;
+ }
+ __atomic_clear(&(atom)->lock,
+ (ret_succ ? success : failure) == _ODP_MEMMODEL_SC ?
+ __ATOMIC_SEQ_CST : __ATOMIC_RELEASE);
+ return ret_succ;
+#endif
+}
+
+/**
+ * Non-atomically get the tag from the tagged pointer datastructure
+ *
+ * @param tptr The tagged pointer to extract the tag from
+ *
+ * @retval Current value of the tag
+ */
+static inline uint64_t _odp_atomic_tptr_gettag(_odp_atomic_tptr_t *tptr)
+{
+ return tptr->tptr.tag;
+}
+
+/**
+ * Non-atomically update the tag of the tagged pointer datastructure
+ *
+ * @param tptr The tagged pointer datastructure to set the tag on
+ * @param tag The value to set the tag to
+ */
+static inline void _odp_atomic_tptr_settag(_odp_atomic_tptr_t *tptr,
+ uint64_t tag)
+{
+ tptr->tptr.tag = tag;
+}
+
+/**
+ * Non-atomically extract the pointer contained in the tagged pointer
+ * datastructure
+ *
+ * @param tptr The tagged pointer datastructure to extract the pointer from
+ *
+ * @return Current pointer value
+ */
+static inline void *_odp_atomic_tptr_getptr(_odp_atomic_tptr_t *tptr)
+{
+ return tptr->tptr.ptr;
+}
+
+/**
+ * Non-atomically set the pointer value contained in the tagged pointer
+ *
+ * @param tptr The tagged pointer datastructure to set the pointer to
+ * @param ptr The pointer value to set
+ */
+static inline void _odp_atomic_tptr_setptr(_odp_atomic_tptr_t *tptr,
+ void *ptr)
+{
+ tptr->tptr.ptr = ptr;
+}
+
+/*****************************************************************************
* Operations on flag atomics
* _odp_atomic_flag_init - no return value
* _odp_atomic_flag_load - return current value
--
1.9.1
_______________________________________________
lng-odp mailing list
lng-odp@lists.linaro.org
http://lists.linaro.org/mailman/listinfo/lng-odp
