duplicated from the API side
(hash: 77a27212b563299959fb2a609b86bb9117a8f918.)
Signed-off-by: Christophe Milard <christophe.mil...@linaro.org>
---
include/odp_drv.h | 1 +
platform/linux-generic/Makefile.am | 3 +
platform/linux-generic/drv_atomic.c | 26 ++
platform/linux-generic/include/odp/drv/atomic.h | 430 +++++++++++++++++++++
.../include/odp/drv/plat/atomic_types.h | 88 +++++
5 files changed, 548 insertions(+)
create mode 100644 platform/linux-generic/drv_atomic.c
create mode 100644 platform/linux-generic/include/odp/drv/atomic.h
create mode 100644 platform/linux-generic/include/odp/drv/plat/atomic_types.h
diff --git a/include/odp_drv.h b/include/odp_drv.h
index 6d2f7ff..35e070c 100644
--- a/include/odp_drv.h
+++ b/include/odp_drv.h
@@ -19,6 +19,7 @@ extern C {
#endif
#include <odp/drv/align.h>
+#include <odp/drv/atomic.h>
#include <odp/drv/byteorder.h>
#include <odp/drv/compiler.h>
#include <odp/drv/std_types.h>
diff --git a/platform/linux-generic/Makefile.am
b/platform/linux-generic/Makefile.am
index 20b4b29..109b42e 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -94,6 +94,7 @@ odpapiplatinclude_HEADERS = \
odpdrvincludedir = $(includedir)/odp/drv
odpdrvinclude_HEADERS = \
$(srcdir)/include/odp/drv/align.h \
+ $(srcdir)/include/odp/drv/atomic.h \
$(srcdir)/include/odp/drv/byteorder.h \
$(srcdir)/include/odp/drv/compiler.h \
$(srcdir)/include/odp/drv/std_types.h \
@@ -101,6 +102,7 @@ odpdrvinclude_HEADERS = \
odpdrvplatincludedir = $(includedir)/odp/drv/plat
odpdrvplatinclude_HEADERS = \
+ $(srcdir)/include/odp/drv/plat/atomic_types.h \
$(srcdir)/include/odp/drv/plat/byteorder_types.h
noinst_HEADERS = \
@@ -198,6 +200,7 @@ __LIB__libodp_linux_la_SOURCES = \
odp_traffic_mngr.c \
odp_version.c \
odp_weak.c \
+ drv_atomic.c \
arch/@ARCH_DIR@/odp_cpu_arch.c \
arch/@ARCH_DIR@/odp_sysinfo_parse.c
diff --git a/platform/linux-generic/drv_atomic.c
b/platform/linux-generic/drv_atomic.c
new file mode 100644
index 0000000..72c85e8
--- /dev/null
+++ b/platform/linux-generic/drv_atomic.c
@@ -0,0 +1,26 @@
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/drv/atomic.h>
+
+int odpdrv_atomic_lock_free_u64(odpdrv_atomic_op_t *atomic_op)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ /* All operations have locks */
+ if (atomic_op)
+ atomic_op->all_bits = 0;
+
+ return 0;
+#else
+ /* All operations are lock-free */
+ if (atomic_op) {
+ atomic_op->all_bits = ~((uint32_t)0);
+ atomic_op->op.init = 0;
+ }
+
+ return 2;
+#endif
+}
diff --git a/platform/linux-generic/include/odp/drv/atomic.h
b/platform/linux-generic/include/odp/drv/atomic.h
new file mode 100644
index 0000000..7d922da
--- /dev/null
+++ b/platform/linux-generic/include/odp/drv/atomic.h
@@ -0,0 +1,430 @@
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODPDRV atomic operations
+ */
+
+#ifndef ODPDRV_PLAT_ATOMIC_H_
+#define ODPDRV_PLAT_ATOMIC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/drv/align.h>
+#include <odp/drv/plat/atomic_types.h>
+
+/** @ingroup odpdrv_atomic
+ * @{
+ */
+
+static inline void odpdrv_atomic_init_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_load_u32(odpdrv_atomic_u32_t *atom)
+{
+ return __atomic_load_n(&atom->v, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_store_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_add_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_add_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_sub_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_sub_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_inc_u32(odpdrv_atomic_u32_t *atom)
+{
+ return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_inc_u32(odpdrv_atomic_u32_t *atom)
+{
+ (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_dec_u32(odpdrv_atomic_u32_t *atom)
+{
+ return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_dec_u32(odpdrv_atomic_u32_t *atom)
+{
+ (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline int odpdrv_atomic_cas_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t *old_val, uint32_t new_val)
+{
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_RELAXED,
+ __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_xchg_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t new_val)
+{
+ return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_max_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t new_max)
+{
+ uint32_t old_val;
+
+ old_val = odpdrv_atomic_load_u32(atom);
+
+ while (new_max > old_val) {
+ if (odpdrv_atomic_cas_u32(atom, &old_val, new_max))
+ break;
+ }
+}
+
+static inline void odpdrv_atomic_min_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t new_min)
+{
+ uint32_t old_val;
+
+ old_val = odpdrv_atomic_load_u32(atom);
+
+ while (new_min < old_val) {
+ if (odpdrv_atomic_cas_u32(atom, &old_val, new_min))
+ break;
+ }
+}
+
+static inline void odpdrv_atomic_init_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+ atom->v = val;
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ __atomic_clear(&atom->lock, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_load_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, (void)0);
+#else
+ return __atomic_load_n(&atom->v, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_store_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v = val);
+#else
+ __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_add_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, atom->v += val);
+#else
+ return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_add_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v += val);
+#else
+ (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_sub_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, atom->v -= val);
+#else
+ return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_sub_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v -= val);
+#else
+ (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_inc_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, atom->v++);
+#else
+ return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_inc_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v++);
+#else
+ (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_dec_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, atom->v--);
+#else
+ return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_dec_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v--);
+#else
+ (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t *old_val, uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ int ret;
+ *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+ return ret;
+#else
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_RELAXED,
+ __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_xchg_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, atom->v = new_val);
+#else
+ return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_max_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t new_max)
+{
+ uint64_t old_val;
+
+ old_val = odpdrv_atomic_load_u64(atom);
+
+ while (new_max > old_val) {
+ if (odpdrv_atomic_cas_u64(atom, &old_val, new_max))
+ break;
+ }
+}
+
+static inline void odpdrv_atomic_min_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t new_min)
+{
+ uint64_t old_val;
+
+ old_val = odpdrv_atomic_load_u64(atom);
+
+ while (new_min < old_val) {
+ if (odpdrv_atomic_cas_u64(atom, &old_val, new_min))
+ break;
+ }
+}
+
+static inline uint32_t odpdrv_atomic_load_acq_u32(odpdrv_atomic_u32_t *atom)
+{
+ return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE);
+}
+
+static inline void odpdrv_atomic_store_rel_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE);
+}
+
+static inline void odpdrv_atomic_add_rel_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE);
+}
+
+static inline void odpdrv_atomic_sub_rel_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t val)
+{
+ (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE);
+}
+
+static inline int odpdrv_atomic_cas_acq_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t *old_val, uint32_t new_val)
+{
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_ACQUIRE,
+ __ATOMIC_RELAXED);
+}
+
+static inline int odpdrv_atomic_cas_rel_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t *old_val, uint32_t new_val)
+{
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_RELEASE,
+ __ATOMIC_RELAXED);
+}
+
+static inline int odpdrv_atomic_cas_acq_rel_u32(odpdrv_atomic_u32_t *atom,
+ uint32_t *old_val,
+ uint32_t new_val)
+{
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_ACQ_REL,
+ __ATOMIC_RELAXED);
+}
+
+static inline uint64_t odpdrv_atomic_load_acq_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ return ATOMIC_OP(atom, (void)0);
+#else
+ return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE);
+#endif
+}
+
+static inline void odpdrv_atomic_store_rel_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v = val);
+#else
+ __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline void odpdrv_atomic_add_rel_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v += val);
+#else
+ (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline void odpdrv_atomic_sub_rel_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ (void)ATOMIC_OP(atom, atom->v -= val);
+#else
+ (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_acq_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t *old_val, uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ int ret;
+ *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+ return ret;
+#else
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_ACQUIRE,
+ __ATOMIC_RELAXED);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_rel_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t *old_val, uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ int ret;
+ *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+ return ret;
+#else
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_RELEASE,
+ __ATOMIC_RELAXED);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_acq_rel_u64(odpdrv_atomic_u64_t *atom,
+ uint64_t *old_val,
+ uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ int ret;
+ *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+ return ret;
+#else
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_ACQ_REL,
+ __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @}
+ */
+
+#include <odp/drv/spec/atomic.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/drv/plat/atomic_types.h
b/platform/linux-generic/include/odp/drv/plat/atomic_types.h
new file mode 100644
index 0000000..6a7ff0d
--- /dev/null
+++ b/platform/linux-generic/include/odp/drv/plat/atomic_types.h
@@ -0,0 +1,88 @@
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODPDRV atomic operations
+ */
+
+#ifndef ODPDRV_ATOMIC_TYPES_H_
+#define ODPDRV_ATOMIC_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/drv/std_types.h>
+#include <odp/drv/align.h>
+
+/**
+ * @internal
+ * Atomic 64-bit unsigned integer
+ */
+struct odpdrv_atomic_u64_s {
+ uint64_t v; /**< Actual storage for the atomic variable */
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+ /* Some architectures do not support lock-free operations on 64-bit
+ * data types. We use a spin lock to ensure atomicity. */
+ char lock; /**< Spin lock (if needed) used to ensure atomic access */
+#endif
+} ODPDRV_ALIGNED(sizeof(uint64_t)); /* Enforce alignement! */;
+
+/**
+ * @internal
+ * Atomic 32-bit unsigned integer
+ */
+struct odpdrv_atomic_u32_s {
+ uint32_t v; /**< Actual storage for the atomic variable */
+} ODPDRV_ALIGNED(sizeof(uint32_t)); /* Enforce alignement! */;
+
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+
+/**
+ * @internal
+ * CAS operation expression for the ATOMIC_OP macro
+ */
+#define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \
+({ \
+ if (atom->v == (old_val)) { \
+ atom->v = (new_val); \
+ *(ret_ptr) = 1; \
+ } else { \
+ *(ret_ptr) = 0; \
+ } \
+})
+
+/**
+ * @internal
+ * Helper macro for lock-based atomic operations on 64-bit integers
+ * @param[in,out] atom Pointer to the 64-bit atomic variable
+ * @param expr Expression used update the variable.
+ * @return The old value of the variable.
+ */
+#define ATOMIC_OP(atom, expr) \
+({ \
+ uint64_t _old_val; \
+ /* Loop while lock is already taken, stop when lock becomes clear */ \
+ while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \
+ (void)0; \
+ _old_val = (atom)->v; \
+ (expr); /* Perform whatever update is desired */ \
+ __atomic_clear(&(atom)->lock, __ATOMIC_RELEASE); \
+ _old_val; /* Return old value */ \
+})
+#endif
+
+typedef struct odpdrv_atomic_u64_s odpdrv_atomic_u64_t;
+
+typedef struct odpdrv_atomic_u32_s odpdrv_atomic_u32_t;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--
2.7.4
