Author: hselasky
Date: Wed Aug 1 12:49:51 2018
New Revision: 337041
URL: https://svnweb.freebsd.org/changeset/base/337041
Log:
MFC r311169, r312975, r313080, r315718 and r316031:
Add support for atomic_(f)cmpset to x86.
Modified:
stable/10/sys/amd64/include/atomic.h
stable/10/sys/i386/include/atomic.h
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/amd64/include/atomic.h
==============================================================================
--- stable/10/sys/amd64/include/atomic.h Wed Aug 1 12:18:52 2018
(r337040)
+++ stable/10/sys/amd64/include/atomic.h Wed Aug 1 12:49:51 2018
(r337041)
@@ -78,8 +78,15 @@
void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \
void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
+int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src);
+int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src);
int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src);
+int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src);
+int atomic_fcmpset_short(volatile u_short *dst, u_short *expect,
+ u_short src);
+int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src);
+int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src);
u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
u_long atomic_fetchadd_long(volatile u_long *p, u_long v);
int atomic_testandset_int(volatile u_int *p, u_int v);
@@ -130,49 +137,62 @@ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##
struct __hack
/*
- * Atomic compare and set, used by the mutex functions
+ * Atomic compare and set, used by the mutex functions.
*
- * if (*dst == expect) *dst = src (all 32 bit words)
+ * cmpset:
+ * if (*dst == expect)
+ * *dst = src
*
- * Returns 0 on failure, non-zero on success
+ * fcmpset:
+ * if (*dst == *expect)
+ * *dst = src
+ * else
+ * *expect = *dst
+ *
+ * Returns 0 on failure, non-zero on success.
*/
-
-static __inline int
-atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src)
-{
- u_char res;
-
- __asm __volatile(
- " " MPLOCKED " "
- " cmpxchgl %3,%1 ; "
- " sete %0 ; "
- "# atomic_cmpset_int"
- : "=q" (res), /* 0 */
- "+m" (*dst), /* 1 */
- "+a" (expect) /* 2 */
- : "r" (src) /* 3 */
- : "memory", "cc");
- return (res);
+#define ATOMIC_CMPSET(TYPE) \
+static __inline int \
+atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \
+{ \
+ u_char res; \
+ \
+ __asm __volatile( \
+ " " MPLOCKED " " \
+ " cmpxchg %3,%1 ; " \
+ " sete %0 ; " \
+ "# atomic_cmpset_" #TYPE " " \
+ : "=q" (res), /* 0 */ \
+ "+m" (*dst), /* 1 */ \
+ "+a" (expect) /* 2 */ \
+ : "r" (src) /* 3 */ \
+ : "memory", "cc"); \
+ return (res); \
+} \
+ \
+static __inline int \
+atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \
+{ \
+ u_char res; \
+ \
+ __asm __volatile( \
+ " " MPLOCKED " " \
+ " cmpxchg %3,%1 ; " \
+ " sete %0 ; " \
+ "# atomic_fcmpset_" #TYPE " " \
+ : "=q" (res), /* 0 */ \
+ "+m" (*dst), /* 1 */ \
+ "+a" (*expect) /* 2 */ \
+ : "r" (src) /* 3 */ \
+ : "memory", "cc"); \
+ return (res); \
}
-static __inline int
-atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
-{
- u_char res;
+ATOMIC_CMPSET(char);
+ATOMIC_CMPSET(short);
+ATOMIC_CMPSET(int);
+ATOMIC_CMPSET(long);
- __asm __volatile(
- " " MPLOCKED " "
- " cmpxchgq %3,%1 ; "
- " sete %0 ; "
- "# atomic_cmpset_long"
- : "=q" (res), /* 0 */
- "+m" (*dst), /* 1 */
- "+a" (expect) /* 2 */
- : "r" (src) /* 3 */
- : "memory", "cc");
- return (res);
-}
-
/*
* Atomically add the value of v to the integer pointed to by p and return
* the previous value of *p.
@@ -409,6 +429,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_add_rel_char atomic_add_barr_char
#define atomic_subtract_acq_char atomic_subtract_barr_char
#define atomic_subtract_rel_char atomic_subtract_barr_char
+#define atomic_cmpset_acq_char atomic_cmpset_char
+#define atomic_cmpset_rel_char atomic_cmpset_char
+#define atomic_fcmpset_acq_char atomic_fcmpset_char
+#define atomic_fcmpset_rel_char atomic_fcmpset_char
#define atomic_set_acq_short atomic_set_barr_short
#define atomic_set_rel_short atomic_set_barr_short
@@ -418,6 +442,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_add_rel_short atomic_add_barr_short
#define atomic_subtract_acq_short atomic_subtract_barr_short
#define atomic_subtract_rel_short atomic_subtract_barr_short
+#define atomic_cmpset_acq_short atomic_cmpset_short
+#define atomic_cmpset_rel_short atomic_cmpset_short
+#define atomic_fcmpset_acq_short atomic_fcmpset_short
+#define atomic_fcmpset_rel_short atomic_fcmpset_short
#define atomic_set_acq_int atomic_set_barr_int
#define atomic_set_rel_int atomic_set_barr_int
@@ -429,6 +457,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_int atomic_subtract_barr_int
#define atomic_cmpset_acq_int atomic_cmpset_int
#define atomic_cmpset_rel_int atomic_cmpset_int
+#define atomic_fcmpset_acq_int atomic_fcmpset_int
+#define atomic_fcmpset_rel_int atomic_fcmpset_int
#define atomic_set_acq_long atomic_set_barr_long
#define atomic_set_rel_long atomic_set_barr_long
@@ -440,6 +470,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_long atomic_subtract_barr_long
#define atomic_cmpset_acq_long atomic_cmpset_long
#define atomic_cmpset_rel_long atomic_cmpset_long
+#define atomic_fcmpset_acq_long atomic_fcmpset_long
+#define atomic_fcmpset_rel_long atomic_fcmpset_long
#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
@@ -459,6 +491,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_8 atomic_subtract_rel_char
#define atomic_load_acq_8 atomic_load_acq_char
#define atomic_store_rel_8 atomic_store_rel_char
+#define atomic_cmpset_8 atomic_cmpset_char
+#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
+#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
+#define atomic_fcmpset_8 atomic_fcmpset_char
+#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
+#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
/* Operations on 16-bit words. */
#define atomic_set_16 atomic_set_short
@@ -475,6 +513,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_16 atomic_subtract_rel_short
#define atomic_load_acq_16 atomic_load_acq_short
#define atomic_store_rel_16 atomic_store_rel_short
+#define atomic_cmpset_16 atomic_cmpset_short
+#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
+#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
+#define atomic_fcmpset_16 atomic_fcmpset_short
+#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
+#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
/* Operations on 32-bit double words. */
#define atomic_set_32 atomic_set_int
@@ -494,6 +538,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_cmpset_32 atomic_cmpset_int
#define atomic_cmpset_acq_32 atomic_cmpset_acq_int
#define atomic_cmpset_rel_32 atomic_cmpset_rel_int
+#define atomic_fcmpset_32 atomic_fcmpset_int
+#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
+#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
#define atomic_swap_32 atomic_swap_int
#define atomic_readandclear_32 atomic_readandclear_int
#define atomic_fetchadd_32 atomic_fetchadd_int
@@ -518,6 +565,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_cmpset_64 atomic_cmpset_long
#define atomic_cmpset_acq_64 atomic_cmpset_acq_long
#define atomic_cmpset_rel_64 atomic_cmpset_rel_long
+#define atomic_fcmpset_64 atomic_fcmpset_long
+#define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long
+#define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long
#define atomic_swap_64 atomic_swap_long
#define atomic_readandclear_64 atomic_readandclear_long
#define atomic_fetchadd_64 atomic_fetchadd_long
@@ -542,6 +592,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_cmpset_ptr atomic_cmpset_long
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
+#define atomic_fcmpset_ptr atomic_fcmpset_long
+#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long
+#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long
#define atomic_swap_ptr atomic_swap_long
#define atomic_readandclear_ptr atomic_readandclear_long
Modified: stable/10/sys/i386/include/atomic.h
==============================================================================
--- stable/10/sys/i386/include/atomic.h Wed Aug 1 12:18:52 2018
(r337040)
+++ stable/10/sys/i386/include/atomic.h Wed Aug 1 12:49:51 2018
(r337041)
@@ -83,7 +83,13 @@
void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \
void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
+int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src);
+int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src);
int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
+int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src);
+int atomic_fcmpset_short(volatile u_short *dst, u_short *expect,
+ u_short src);
+int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src);
u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
int atomic_testandset_int(volatile u_int *p, u_int v);
int atomic_testandclear_int(volatile u_int *p, u_int v);
@@ -137,60 +143,61 @@ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##
struct __hack
/*
- * Atomic compare and set, used by the mutex functions
+ * Atomic compare and set, used by the mutex functions.
*
- * if (*dst == expect) *dst = src (all 32 bit words)
+ * cmpset:
+ * if (*dst == expect)
+ * *dst = src
*
- * Returns 0 on failure, non-zero on success
+ * fcmpset:
+ * if (*dst == *expect)
+ * *dst = src
+ * else
+ * *expect = *dst
+ *
+ * Returns 0 on failure, non-zero on success.
*/
-
-#ifdef CPU_DISABLE_CMPXCHG
-
-static __inline int
-atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src)
-{
- u_char res;
-
- __asm __volatile(
- " pushfl ; "
- " cli ; "
- " cmpl %3,%1 ; "
- " jne 1f ; "
- " movl %2,%1 ; "
- "1: "
- " sete %0 ; "
- " popfl ; "
- "# atomic_cmpset_int"
- : "=q" (res), /* 0 */
- "+m" (*dst) /* 1 */
- : "r" (src), /* 2 */
- "r" (expect) /* 3 */
- : "memory");
- return (res);
+#define ATOMIC_CMPSET(TYPE, CONS) \
+static __inline int \
+atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \
+{ \
+ u_char res; \
+ \
+ __asm __volatile( \
+ " " MPLOCKED " " \
+ " cmpxchg %3,%1 ; " \
+ " sete %0 ; " \
+ "# atomic_cmpset_" #TYPE " " \
+ : "=q" (res), /* 0 */ \
+ "+m" (*dst), /* 1 */ \
+ "+a" (expect) /* 2 */ \
+ : CONS (src) /* 3 */ \
+ : "memory", "cc"); \
+ return (res); \
+} \
+ \
+static __inline int \
+atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \
+{ \
+ u_char res; \
+ \
+ __asm __volatile( \
+ " " MPLOCKED " " \
+ " cmpxchg %3,%1 ; " \
+ " sete %0 ; " \
+ "# atomic_fcmpset_" #TYPE " " \
+ : "=q" (res), /* 0 */ \
+ "+m" (*dst), /* 1 */ \
+ "+a" (*expect) /* 2 */ \
+ : CONS (src) /* 3 */ \
+ : "memory", "cc"); \
+ return (res); \
}
-#else /* !CPU_DISABLE_CMPXCHG */
+ATOMIC_CMPSET(char, "q");
+ATOMIC_CMPSET(short, "r");
+ATOMIC_CMPSET(int, "r");
-static __inline int
-atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src)
-{
- u_char res;
-
- __asm __volatile(
- " " MPLOCKED " "
- " cmpxchgl %3,%1 ; "
- " sete %0 ; "
- "# atomic_cmpset_int"
- : "=q" (res), /* 0 */
- "+m" (*dst), /* 1 */
- "+a" (expect) /* 2 */
- : "r" (src) /* 3 */
- : "memory", "cc");
- return (res);
-}
-
-#endif /* CPU_DISABLE_CMPXCHG */
-
/*
* Atomically add the value of v to the integer pointed to by p and return
* the previous value of *p.
@@ -623,6 +630,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_add_rel_char atomic_add_barr_char
#define atomic_subtract_acq_char atomic_subtract_barr_char
#define atomic_subtract_rel_char atomic_subtract_barr_char
+#define atomic_cmpset_acq_char atomic_cmpset_char
+#define atomic_cmpset_rel_char atomic_cmpset_char
+#define atomic_fcmpset_acq_char atomic_fcmpset_char
+#define atomic_fcmpset_rel_char atomic_fcmpset_char
#define atomic_set_acq_short atomic_set_barr_short
#define atomic_set_rel_short atomic_set_barr_short
@@ -632,6 +643,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_add_rel_short atomic_add_barr_short
#define atomic_subtract_acq_short atomic_subtract_barr_short
#define atomic_subtract_rel_short atomic_subtract_barr_short
+#define atomic_cmpset_acq_short atomic_cmpset_short
+#define atomic_cmpset_rel_short atomic_cmpset_short
+#define atomic_fcmpset_acq_short atomic_fcmpset_short
+#define atomic_fcmpset_rel_short atomic_fcmpset_short
#define atomic_set_acq_int atomic_set_barr_int
#define atomic_set_rel_int atomic_set_barr_int
@@ -643,6 +658,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_int atomic_subtract_barr_int
#define atomic_cmpset_acq_int atomic_cmpset_int
#define atomic_cmpset_rel_int atomic_cmpset_int
+#define atomic_fcmpset_acq_int atomic_fcmpset_int
+#define atomic_fcmpset_rel_int atomic_fcmpset_int
#define atomic_set_acq_long atomic_set_barr_long
#define atomic_set_rel_long atomic_set_barr_long
@@ -654,6 +671,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_long atomic_subtract_barr_long
#define atomic_cmpset_acq_long atomic_cmpset_long
#define atomic_cmpset_rel_long atomic_cmpset_long
+#define atomic_fcmpset_acq_long atomic_fcmpset_long
+#define atomic_fcmpset_rel_long atomic_fcmpset_long
#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
@@ -673,6 +692,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_8 atomic_subtract_rel_char
#define atomic_load_acq_8 atomic_load_acq_char
#define atomic_store_rel_8 atomic_store_rel_char
+#define atomic_cmpset_8 atomic_cmpset_char
+#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
+#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
+#define atomic_fcmpset_8 atomic_fcmpset_char
+#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
+#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
/* Operations on 16-bit words. */
#define atomic_set_16 atomic_set_short
@@ -689,6 +714,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_subtract_rel_16 atomic_subtract_rel_short
#define atomic_load_acq_16 atomic_load_acq_short
#define atomic_store_rel_16 atomic_store_rel_short
+#define atomic_cmpset_16 atomic_cmpset_short
+#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
+#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
+#define atomic_fcmpset_16 atomic_fcmpset_short
+#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
+#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
/* Operations on 32-bit double words. */
#define atomic_set_32 atomic_set_int
@@ -708,6 +739,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_cmpset_32 atomic_cmpset_int
#define atomic_cmpset_acq_32 atomic_cmpset_acq_int
#define atomic_cmpset_rel_32 atomic_cmpset_rel_int
+#define atomic_fcmpset_32 atomic_fcmpset_int
+#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
+#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
#define atomic_swap_32 atomic_swap_int
#define atomic_readandclear_32 atomic_readandclear_int
#define atomic_fetchadd_32 atomic_fetchadd_int
@@ -750,6 +784,14 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
(u_int)(new))
#define atomic_cmpset_rel_ptr(dst, old, new) \
atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \
+ (u_int)(new))
+#define atomic_fcmpset_ptr(dst, old, new) \
+ atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old),
(u_int)(new))
+#define atomic_fcmpset_acq_ptr(dst, old, new) \
+ atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \
+ (u_int)(new))
+#define atomic_fcmpset_rel_ptr(dst, old, new) \
+ atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \
(u_int)(new))
#define atomic_swap_ptr(p, v) \
atomic_swap_int((volatile u_int *)(p), (u_int)(v))
_______________________________________________
svn-src-stable-10@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-stable-10
To unsubscribe, send any mail to "svn-src-stable-10-unsubscr...@freebsd.org"
