Author: ian
Date: Tue Oct 1 19:39:00 2019
New Revision: 352938
URL: https://svnweb.freebsd.org/changeset/base/352938
Log:
Add 8 and 16 bit versions of atomic_cmpset and atomic_fcmpset for arm.
This adds 8 and 16 bit versions of the cmpset and fcmpset functions. Macros
are used to generate all the flavors from the same set of instructions; the
macro expansion handles the couple minor differences between each size
variation (generating ldrexb/ldrexh/ldrex for 8/16/32, etc).
In addition to handling new sizes, the instruction sequences used for cmpset
and fcmpset are rewritten to be a bit shorter/faster, and the new sequence
will not return false when *dst==*old but the store-exclusive fails because
of concurrent writers. Instead, it just loops like ldrex/strex sequences
normally do until it gets a non-conflicted store. The manpage allows LL/SC
architectures to bogusly return false, but there's no reason to actually do
so, at least on arm.
Reviewed by: cognet
Modified:
head/sys/arm/include/atomic-v4.h
head/sys/arm/include/atomic-v6.h
Modified: head/sys/arm/include/atomic-v4.h
==============================================================================
--- head/sys/arm/include/atomic-v4.h Tue Oct 1 18:32:27 2019
(r352937)
+++ head/sys/arm/include/atomic-v4.h Tue Oct 1 19:39:00 2019
(r352938)
@@ -113,6 +113,43 @@ atomic_clear_64(volatile uint64_t *address, uint64_t c
}
static __inline int
+atomic_fcmpset_8(volatile uint8_t *p, volatile uint8_t *cmpval, volatile
uint8_t newval)
+{
+ int ret;
+
+ __with_interrupts_disabled(
+ {
+ ret = *p;
+ if (*p == *cmpval) {
+ *p = newval;
+ ret = 1;
+ } else {
+ *cmpval = *p;
+ ret = 0;
+ }
+ });
+ return (ret);
+}
+static __inline int
+atomic_fcmpset_16(volatile uint16_t *p, volatile uint16_t *cmpval, volatile
uint16_t newval)
+{
+ int ret;
+
+ __with_interrupts_disabled(
+ {
+ ret = *p;
+ if (*p == *cmpval) {
+ *p = newval;
+ ret = 1;
+ } else {
+ *cmpval = *p;
+ ret = 0;
+ }
+ });
+ return (ret);
+}
+
+static __inline int
atomic_fcmpset_32(volatile u_int32_t *p, volatile u_int32_t *cmpval, volatile
u_int32_t newval)
{
int ret;
@@ -150,6 +187,40 @@ atomic_fcmpset_64(volatile u_int64_t *p, volatile u_in
}
static __inline int
+atomic_cmpset_8(volatile uint8_t *p, volatile uint8_t cmpval, volatile uint8_t
newval)
+{
+ int ret;
+
+ __with_interrupts_disabled(
+ {
+ if (*p == cmpval) {
+ *p = newval;
+ ret = 1;
+ } else {
+ ret = 0;
+ }
+ });
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_16(volatile uint16_t *p, volatile uint16_t cmpval, volatile
uint16_t newval)
+{
+ int ret;
+
+ __with_interrupts_disabled(
+ {
+ if (*p == cmpval) {
+ *p = newval;
+ ret = 1;
+ } else {
+ ret = 0;
+ }
+ });
+ return (ret);
+}
+
+static __inline int
atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile
u_int32_t newval)
{
int ret;
@@ -450,6 +521,10 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v)
#define atomic_fcmpset_rel_32 atomic_fcmpset_32
#define atomic_fcmpset_acq_32 atomic_fcmpset_32
#ifdef _KERNEL
+#define atomic_fcmpset_rel_8 atomic_fcmpset_8
+#define atomic_fcmpset_acq_8 atomic_fcmpset_8
+#define atomic_fcmpset_rel_16 atomic_fcmpset_16
+#define atomic_fcmpset_acq_16 atomic_fcmpset_16
#define atomic_fcmpset_rel_64 atomic_fcmpset_64
#define atomic_fcmpset_acq_64 atomic_fcmpset_64
#endif
@@ -458,6 +533,10 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v)
#define atomic_cmpset_rel_32 atomic_cmpset_32
#define atomic_cmpset_acq_32 atomic_cmpset_32
#ifdef _KERNEL
+#define atomic_cmpset_rel_8 atomic_cmpset_8
+#define atomic_cmpset_acq_8 atomic_cmpset_8
+#define atomic_cmpset_rel_16 atomic_cmpset_16
+#define atomic_cmpset_acq_16 atomic_cmpset_16
#define atomic_cmpset_rel_64 atomic_cmpset_64
#define atomic_cmpset_acq_64 atomic_cmpset_64
#endif
Modified: head/sys/arm/include/atomic-v6.h
==============================================================================
--- head/sys/arm/include/atomic-v6.h Tue Oct 1 18:32:27 2019
(r352937)
+++ head/sys/arm/include/atomic-v6.h Tue Oct 1 19:39:00 2019
(r352938)
@@ -190,224 +190,380 @@ ATOMIC_ACQ_REL(clear, 32)
ATOMIC_ACQ_REL(clear, 64)
ATOMIC_ACQ_REL_LONG(clear)
+#define ATOMIC_FCMPSET_CODE(RET, TYPE, SUF) \
+ { \
+ TYPE tmp; \
+ \
+ __asm __volatile( \
+ "1: ldrex" SUF " %[tmp], [%[ptr]] \n" \
+ " ldr %[ret], [%[oldv]] \n" \
+ " teq %[tmp], %[ret] \n" \
+ " ittee ne \n" \
+ " str" SUF "ne %[tmp], [%[oldv]] \n" \
+ " movne %[ret], #0 \n" \
+ " strex" SUF "eq %[ret], %[newv], [%[ptr]] \n" \
+ " eorseq %[ret], #1 \n" \
+ " beq 1b \n" \
+ : [ret] "=&r" (RET), \
+ [tmp] "=&r" (tmp) \
+ : [ptr] "r" (_ptr), \
+ [oldv] "r" (_old), \
+ [newv] "r" (_new) \
+ : "cc", "memory"); \
+ }
+
+#define ATOMIC_FCMPSET_CODE64(RET) \
+ { \
+ uint64_t cmp, tmp; \
+ \
+ __asm __volatile( \
+ "1: ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" \
+ " ldrd %Q[cmp], %R[cmp], [%[oldv]] \n" \
+ " teq %Q[tmp], %Q[cmp] \n" \
+ " it eq \n" \
+ " teqeq %R[tmp], %R[cmp] \n" \
+ " ittee ne \n" \
+ " movne %[ret], #0 \n" \
+ " strdne %[cmp], [%[oldv]] \n" \
+ " strexdeq %[ret], %Q[newv], %R[newv], [%[ptr]] \n" \
+ " eorseq %[ret], #1 \n" \
+ " beq 1b \n" \
+ : [ret] "=&r" (RET), \
+ [cmp] "=&r" (cmp), \
+ [tmp] "=&r" (tmp) \
+ : [ptr] "r" (_ptr), \
+ [oldv] "r" (_old), \
+ [newv] "r" (_new) \
+ : "cc", "memory"); \
+ }
+
static __inline int
-atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+atomic_fcmpset_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new)
{
- uint32_t tmp;
- uint32_t _cmpval = *cmpval;
int ret;
- __asm __volatile(
- " mov %0, #1 \n"
- " ldrex %1, [%2] \n"
- " cmp %1, %3 \n"
- " it eq \n"
- " strexeq %0, %4, [%2] \n"
- : "=&r" (ret), "=&r" (tmp), "+r" (p), "+r" (_cmpval), "+r" (newval)
- : : "cc", "memory");
- *cmpval = tmp;
- return (!ret);
+ ATOMIC_FCMPSET_CODE(ret, uint8_t, "b");
+ return (ret);
}
static __inline int
-atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+atomic_fcmpset_acq_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new)
{
- uint64_t tmp;
- uint64_t _cmpval = *cmpval;
int ret;
- __asm __volatile(
- "1: mov %[ret], #1 \n"
- " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n"
- " teq %Q[tmp], %Q[_cmpval] \n"
- " ite eq \n"
- " teqeq %R[tmp], %R[_cmpval] \n"
- " bne 2f \n"
- " strexd %[ret], %Q[newval], %R[newval], [%[ptr]]\n"
- "2: \n"
- : [ret] "=&r" (ret),
- [tmp] "=&r" (tmp)
- : [ptr] "r" (p),
- [_cmpval] "r" (_cmpval),
- [newval] "r" (newval)
- : "cc", "memory");
- *cmpval = tmp;
- return (!ret);
+ ATOMIC_FCMPSET_CODE(ret, uint8_t, "b");
+ dmb();
+ return (ret);
}
static __inline int
-atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
+atomic_fcmpset_rel_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new)
{
+ int ret;
- return (atomic_fcmpset_32((volatile uint32_t *)p,
- (uint32_t *)cmpval, newval));
+ dmb();
+ ATOMIC_FCMPSET_CODE(ret, uint8_t, "b");
+ return (ret);
}
static __inline int
-atomic_fcmpset_acq_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+atomic_fcmpset_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new)
{
int ret;
- ret = atomic_fcmpset_64(p, cmpval, newval);
+ ATOMIC_FCMPSET_CODE(ret, uint16_t, "h");
+ return (ret);
+}
+
+static __inline int
+atomic_fcmpset_acq_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new)
+{
+ int ret;
+
+ ATOMIC_FCMPSET_CODE(ret, uint16_t, "h");
dmb();
return (ret);
}
static __inline int
-atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval)
+atomic_fcmpset_rel_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new)
{
int ret;
- ret = atomic_fcmpset_long(p, cmpval, newval);
dmb();
+ ATOMIC_FCMPSET_CODE(ret, uint16_t, "h");
return (ret);
}
static __inline int
-atomic_fcmpset_acq_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+atomic_fcmpset_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new)
{
+ int ret;
+ ATOMIC_FCMPSET_CODE(ret, uint32_t, "");
+ return (ret);
+}
+
+static __inline int
+atomic_fcmpset_acq_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new)
+{
int ret;
- ret = atomic_fcmpset_32(p, cmpval, newval);
+ ATOMIC_FCMPSET_CODE(ret, uint32_t, "");
dmb();
return (ret);
}
static __inline int
-atomic_fcmpset_rel_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
+atomic_fcmpset_rel_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new)
{
+ int ret;
dmb();
- return (atomic_fcmpset_32(p, cmpval, newval));
+ ATOMIC_FCMPSET_CODE(ret, uint32_t, "");
+ return (ret);
}
static __inline int
-atomic_fcmpset_rel_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
+atomic_fcmpset_long(volatile long *_ptr, long *_old, long _new)
{
+ int ret;
+ ATOMIC_FCMPSET_CODE(ret, long, "");
+ return (ret);
+}
+
+static __inline int
+atomic_fcmpset_acq_long(volatile long *_ptr, long *_old, long _new)
+{
+ int ret;
+
+ ATOMIC_FCMPSET_CODE(ret, long, "");
dmb();
- return (atomic_fcmpset_64(p, cmpval, newval));
+ return (ret);
}
static __inline int
-atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval)
+atomic_fcmpset_rel_long(volatile long *_ptr, long *_old, long _new)
{
+ int ret;
dmb();
- return (atomic_fcmpset_long(p, cmpval, newval));
+ ATOMIC_FCMPSET_CODE(ret, long, "");
+ return (ret);
}
static __inline int
-atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
+atomic_fcmpset_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new)
{
int ret;
- __asm __volatile(
- "1: ldrex %0, [%1] \n"
- " cmp %0, %2 \n"
- " itt ne \n"
- " movne %0, #0 \n"
- " bne 2f \n"
- " strex %0, %3, [%1] \n"
- " cmp %0, #0 \n"
- " ite eq \n"
- " moveq %0, #1 \n"
- " bne 1b \n"
- "2:"
- : "=&r" (ret), "+r" (p), "+r" (cmpval), "+r" (newval)
- : : "cc", "memory");
+ ATOMIC_FCMPSET_CODE64(ret);
return (ret);
}
static __inline int
-atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
+atomic_fcmpset_acq_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new)
{
- uint64_t tmp;
- uint32_t ret;
+ int ret;
- __asm __volatile(
- "1: \n"
- " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n"
- " teq %Q[tmp], %Q[cmpval] \n"
- " itee eq \n"
- " teqeq %R[tmp], %R[cmpval] \n"
- " movne %[ret], #0 \n"
- " bne 2f \n"
- " strexd %[ret], %Q[newval], %R[newval], [%[ptr]]\n"
- " teq %[ret], #0 \n"
- " it ne \n"
- " bne 1b \n"
- " mov %[ret], #1 \n"
- "2: \n"
- : [ret] "=&r" (ret),
- [tmp] "=&r" (tmp)
- : [ptr] "r" (p),
- [cmpval] "r" (cmpval),
- [newval] "r" (newval)
- : "cc", "memory");
+ ATOMIC_FCMPSET_CODE64(ret);
+ dmb();
return (ret);
}
static __inline int
-atomic_cmpset_long(volatile u_long *p, u_long cmpval, u_long newval)
+atomic_fcmpset_rel_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new)
{
+ int ret;
- return (atomic_cmpset_32((volatile uint32_t *)p, cmpval, newval));
+ dmb();
+ ATOMIC_FCMPSET_CODE64(ret);
+ return (ret);
}
+#define ATOMIC_CMPSET_CODE(RET, SUF) \
+ { \
+ __asm __volatile( \
+ "1: ldrex" SUF " %[ret], [%[ptr]] \n" \
+ " teq %[ret], %[oldv] \n" \
+ " itee ne \n" \
+ " movne %[ret], #0 \n" \
+ " strex" SUF "eq %[ret], %[newv], [%[ptr]] \n" \
+ " eorseq %[ret], #1 \n" \
+ " beq 1b \n" \
+ : [ret] "=&r" (RET) \
+ : [ptr] "r" (_ptr), \
+ [oldv] "r" (_old), \
+ [newv] "r" (_new) \
+ : "cc", "memory"); \
+ }
+
+#define ATOMIC_CMPSET_CODE64(RET) \
+ { \
+ uint64_t tmp; \
+ \
+ __asm __volatile( \
+ "1: ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" \
+ " teq %Q[tmp], %Q[oldv] \n" \
+ " it eq \n" \
+ " teqeq %R[tmp], %R[oldv] \n" \
+ " itee ne \n" \
+ " movne %[ret], #0 \n" \
+ " strexdeq %[ret], %Q[newv], %R[newv], [%[ptr]] \n" \
+ " eorseq %[ret], #1 \n" \
+ " beq 1b \n" \
+ : [ret] "=&r" (RET), \
+ [tmp] "=&r" (tmp) \
+ : [ptr] "r" (_ptr), \
+ [oldv] "r" (_old), \
+ [newv] "r" (_new) \
+ : "cc", "memory"); \
+ }
+
static __inline int
-atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
+atomic_cmpset_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new)
{
int ret;
- ret = atomic_cmpset_32(p, cmpval, newval);
+ ATOMIC_CMPSET_CODE(ret, "b");
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_acq_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE(ret, "b");
dmb();
return (ret);
}
static __inline int
-atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
+atomic_cmpset_rel_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new)
{
int ret;
- ret = atomic_cmpset_64(p, cmpval, newval);
dmb();
+ ATOMIC_CMPSET_CODE(ret, "b");
return (ret);
}
static __inline int
-atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
+atomic_cmpset_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new)
{
int ret;
- ret = atomic_cmpset_long(p, cmpval, newval);
+ ATOMIC_CMPSET_CODE(ret, "h");
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_acq_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE(ret, "h");
dmb();
return (ret);
}
static __inline int
-atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
+atomic_cmpset_rel_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new)
{
+ int ret;
dmb();
- return (atomic_cmpset_32(p, cmpval, newval));
+ ATOMIC_CMPSET_CODE(ret, "h");
+ return (ret);
}
static __inline int
-atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
+atomic_cmpset_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new)
{
+ int ret;
+ ATOMIC_CMPSET_CODE(ret, "");
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_acq_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE(ret, "");
dmb();
- return (atomic_cmpset_64(p, cmpval, newval));
+ return (ret);
}
static __inline int
-atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
+atomic_cmpset_rel_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new)
{
+ int ret;
dmb();
- return (atomic_cmpset_long(p, cmpval, newval));
+ ATOMIC_CMPSET_CODE(ret, "");
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_long(volatile long *_ptr, long _old, long _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE(ret, "");
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_acq_long(volatile long *_ptr, long _old, long _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE(ret, "");
+ dmb();
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_rel_long(volatile long *_ptr, long _old, long _new)
+{
+ int ret;
+
+ dmb();
+ ATOMIC_CMPSET_CODE(ret, "");
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE64(ret);
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_acq_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new)
+{
+ int ret;
+
+ ATOMIC_CMPSET_CODE64(ret);
+ dmb();
+ return (ret);
+}
+
+static __inline int
+atomic_cmpset_rel_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new)
+{
+ int ret;
+
+ dmb();
+ ATOMIC_CMPSET_CODE64(ret);
+ return (ret);
}
static __inline uint32_t
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
