Module Name: src Committed By: riastradh Date: Mon Aug 27 13:41:08 UTC 2018
Modified Files: src/sys/external/bsd/drm2/include/linux: atomic.h Log Message: Attempt to match Linux semantics for membars implied by atomics. This is kind of moot at the moment because we're mostly x86-only for drmkms, but this might help in the future if we ever went beyond x86. To generate a diff of this commit: cvs rdiff -u -r1.12 -r1.13 src/sys/external/bsd/drm2/include/linux/atomic.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/external/bsd/drm2/include/linux/atomic.h diff -u src/sys/external/bsd/drm2/include/linux/atomic.h:1.12 src/sys/external/bsd/drm2/include/linux/atomic.h:1.13 --- src/sys/external/bsd/drm2/include/linux/atomic.h:1.12 Mon Aug 27 13:40:53 2018 +++ src/sys/external/bsd/drm2/include/linux/atomic.h Mon Aug 27 13:41:08 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: atomic.h,v 1.12 2018/08/27 13:40:53 riastradh Exp $ */ +/* $NetBSD: atomic.h,v 1.13 2018/08/27 13:41:08 riastradh Exp $ */ /*- * Copyright (c) 2013 The NetBSD Foundation, Inc. @@ -36,6 +36,22 @@ #include <machine/limits.h> +#if defined(MULTIPROCESSOR) && !defined(__HAVE_ATOMIC_AS_MEMBAR) +# define smp_mb__before_atomic() membar_exit() +# define smp_mb__after_atomic() membar_enter() +#else +# define smp_mb__before_atomic() __insn_barrier() +# define smp_mb__after_atomic() __insn_barrier() +#endif + +/* + * atomic (u)int operations + * + * Atomics that return a value, other than atomic_read, imply a + * full memory_sync barrier. Those that do not return a value + * imply no memory barrier. + */ + struct atomic { union { volatile int au_int; @@ -50,78 +66,106 @@ typedef struct atomic atomic_t; static inline int atomic_read(atomic_t *atomic) { + /* no membar */ return atomic->a_u.au_int; } static inline void atomic_set(atomic_t *atomic, int value) { + /* no membar */ atomic->a_u.au_int = value; } static inline void atomic_add(int addend, atomic_t *atomic) { + /* no membar */ atomic_add_int(&atomic->a_u.au_uint, addend); } static inline void atomic_sub(int subtrahend, atomic_t *atomic) { + /* no membar */ atomic_add_int(&atomic->a_u.au_uint, -subtrahend); } static inline int atomic_add_return(int addend, atomic_t *atomic) { - return (int)atomic_add_int_nv(&atomic->a_u.au_uint, addend); + int v; + + smp_mb__before_atomic(); + v = (int)atomic_add_int_nv(&atomic->a_u.au_uint, addend); + smp_mb__after_atomic(); + + return v; } static inline void atomic_inc(atomic_t *atomic) { + /* no membar */ atomic_inc_uint(&atomic->a_u.au_uint); } static inline void atomic_dec(atomic_t *atomic) { + /* no membar */ atomic_dec_uint(&atomic->a_u.au_uint); } static inline int atomic_inc_return(atomic_t *atomic) { - return (int)atomic_inc_uint_nv(&atomic->a_u.au_uint); + int v; + + smp_mb__before_atomic(); + v = (int)atomic_inc_uint_nv(&atomic->a_u.au_uint); + smp_mb__after_atomic(); + + return v; } static inline int atomic_dec_return(atomic_t *atomic) { - return (int)atomic_dec_uint_nv(&atomic->a_u.au_uint); + int v; + + smp_mb__before_atomic(); + v = (int)atomic_dec_uint_nv(&atomic->a_u.au_uint); + smp_mb__after_atomic(); + + return v; } static inline int atomic_dec_and_test(atomic_t *atomic) { - return (0 == (int)atomic_dec_uint_nv(&atomic->a_u.au_uint)); + /* membar implied by atomic_dec_return */ + return atomic_dec_return(atomic) == 0; } static inline void atomic_or(int value, atomic_t *atomic) { + /* no membar */ atomic_or_uint(&atomic->a_u.au_uint, value); } static inline void atomic_set_mask(unsigned long mask, atomic_t *atomic) { + /* no membar */ atomic_or_uint(&atomic->a_u.au_uint, mask); } static inline void atomic_clear_mask(unsigned long mask, atomic_t *atomic) { + /* no membar */ atomic_and_uint(&atomic->a_u.au_uint, ~mask); } @@ -130,33 +174,53 @@ atomic_add_unless(atomic_t *atomic, int { int value; + smp_mb__before_atomic(); do { value = atomic->a_u.au_int; if (value == zero) - return 0; + break; } while (atomic_cas_uint(&atomic->a_u.au_uint, value, (value + addend)) != value); + smp_mb__after_atomic(); - return 1; + return value != zero; } static inline int atomic_inc_not_zero(atomic_t *atomic) { + /* membar implied by atomic_add_unless */ return atomic_add_unless(atomic, 1, 0); } static inline int atomic_xchg(atomic_t *atomic, int new) { - return (int)atomic_swap_uint(&atomic->a_u.au_uint, (unsigned)new); + int old; + + smp_mb__before_atomic(); + old = (int)atomic_swap_uint(&atomic->a_u.au_uint, (unsigned)new); + smp_mb__after_atomic(); + + return old; } static inline int -atomic_cmpxchg(atomic_t *atomic, int old, int new) +atomic_cmpxchg(atomic_t *atomic, int expect, int new) { - return (int)atomic_cas_uint(&atomic->a_u.au_uint, (unsigned)old, + int old; + + /* + * XXX As an optimization, under Linux's semantics we are + * allowed to skip the memory barrier if the comparison fails, + * but taking advantage of that is not convenient here. + */ + smp_mb__before_atomic(); + old = (int)atomic_cas_uint(&atomic->a_u.au_uint, (unsigned)expect, (unsigned)new); + smp_mb__after_atomic(); + + return old; } struct atomic64 { @@ -168,37 +232,58 @@ typedef struct atomic64 atomic64_t; static inline uint64_t atomic64_read(const struct atomic64 *a) { + /* no membar */ return a->a_v; } static inline void atomic64_set(struct atomic64 *a, uint64_t v) { + /* no membar */ a->a_v = v; } static inline void atomic64_add(long long d, struct atomic64 *a) { + /* no membar */ atomic_add_64(&a->a_v, d); } static inline void atomic64_sub(long long d, struct atomic64 *a) { + /* no membar */ atomic_add_64(&a->a_v, -d); } static inline uint64_t -atomic64_xchg(struct atomic64 *a, uint64_t v) +atomic64_xchg(struct atomic64 *a, uint64_t new) { - return atomic_swap_64(&a->a_v, v); + uint64_t old; + + smp_mb__before_atomic(); + old = atomic_swap_64(&a->a_v, new); + smp_mb__after_atomic(); + + return old; } static inline uint64_t -atomic64_cmpxchg(struct atomic64 *atomic, uint64_t old, uint64_t new) +atomic64_cmpxchg(struct atomic64 *atomic, uint64_t expect, uint64_t new) { - return atomic_cas_64(&atomic->a_v, old, new); + uint64_t old; + + /* + * XXX As an optimization, under Linux's semantics we are + * allowed to skip the memory barrier if the comparison fails, + * but taking advantage of that is not convenient here. + */ + smp_mb__before_atomic(); + old = atomic_cas_64(&atomic->a_v, expect, new); + smp_mb__after_atomic(); + + return old; } static inline void @@ -206,6 +291,7 @@ set_bit(unsigned int bit, volatile unsig { const unsigned int units = (sizeof(*ptr) * CHAR_BIT); + /* no memory barrier */ atomic_or_ulong(&ptr[bit / units], (1UL << (bit % units))); } @@ -214,6 +300,7 @@ clear_bit(unsigned int bit, volatile uns { const unsigned int units = (sizeof(*ptr) * CHAR_BIT); + /* no memory barrier */ atomic_and_ulong(&ptr[bit / units], ~(1UL << (bit % units))); } @@ -225,6 +312,7 @@ change_bit(unsigned int bit, volatile un const unsigned long mask = (1UL << (bit % units)); unsigned long v; + /* no memory barrier */ do v = *p; while (atomic_cas_ulong(p, v, (v ^ mask)) != v); } @@ -236,7 +324,9 @@ test_and_set_bit(unsigned int bit, volat const unsigned long mask = (1UL << (bit % units)); unsigned long v; + smp_mb__before_atomic(); do v = *p; while (atomic_cas_ulong(p, v, (v | mask)) != v); + smp_mb__after_atomic(); return ((v & mask) != 0); } @@ -249,7 +339,9 @@ test_and_clear_bit(unsigned int bit, vol const unsigned long mask = (1UL << (bit % units)); unsigned long v; + smp_mb__before_atomic(); do v = *p; while (atomic_cas_ulong(p, v, (v & ~mask)) != v); + smp_mb__after_atomic(); return ((v & mask) != 0); } @@ -262,24 +354,11 @@ test_and_change_bit(unsigned int bit, vo const unsigned long mask = (1UL << (bit % units)); unsigned long v; + smp_mb__before_atomic(); do v = *p; while (atomic_cas_ulong(p, v, (v ^ mask)) != v); + smp_mb__after_atomic(); return ((v & mask) != 0); } -#if defined(MULTIPROCESSOR) && !defined(__HAVE_ATOMIC_AS_MEMBAR) -/* - * XXX These memory barriers are doubtless overkill, but I am having - * trouble understanding the intent and use of the Linux atomic membar - * API. I think that for reference counting purposes, the sequences - * should be insn/inc/enter and exit/dec/insn, but the use of the - * before/after memory barriers is not consistent throughout Linux. - */ -# define smp_mb__before_atomic() membar_sync() -# define smp_mb__after_atomic() membar_sync() -#else -# define smp_mb__before_atomic() __insn_barrier() -# define smp_mb__after_atomic() __insn_barrier() -#endif - #endif /* _LINUX_ATOMIC_H_ */