Module Name: src
Committed By: martin
Date: Wed Aug 11 17:19:01 UTC 2021
Modified Files:
src/common/lib/libc/arch/aarch64/atomic [netbsd-9]: atomic_add_16.S
atomic_add_32.S atomic_add_64.S atomic_add_8.S atomic_and_16.S
atomic_and_32.S atomic_and_64.S atomic_and_8.S atomic_cas_16.S
atomic_cas_32.S atomic_cas_64.S atomic_cas_8.S atomic_dec_32.S
atomic_dec_64.S atomic_inc_32.S atomic_inc_64.S atomic_nand_16.S
atomic_nand_32.S atomic_nand_64.S atomic_nand_8.S atomic_op_asm.h
atomic_or_16.S atomic_or_32.S atomic_or_64.S atomic_or_8.S
atomic_sub_16.S atomic_sub_32.S atomic_sub_64.S atomic_sub_8.S
atomic_swap_16.S atomic_swap_32.S atomic_swap_64.S atomic_swap_8.S
atomic_xor_16.S atomic_xor_32.S atomic_xor_64.S atomic_xor_8.S
membar_ops.S
Log Message:
Pull up following revision(s) (requested by skrll in ticket #1331):
common/lib/libc/arch/aarch64/atomic/atomic_add_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_or_64.S: revision 1.3
common/lib/libc/arch/aarch64/atomic/atomic_and_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_and_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_add_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h: revision 1.6
common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S: revision 1.4
common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S: revision 1.6
common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_or_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_or_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_or_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S: revision 1.4
common/lib/libc/arch/aarch64/atomic/atomic_add_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S: revision 1.4
common/lib/libc/arch/aarch64/atomic/atomic_add_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S: revision 1.4
common/lib/libc/arch/aarch64/atomic/membar_ops.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S: revision 1.5
common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_and_8.S: revision 1.2
common/lib/libc/arch/aarch64/atomic/atomic_and_64.S: revision 1.2
Part I of ad@'s performance improvements for aarch64
- Remove memory barriers from the atomic ops. I don't understand why thos=
e
are there. Is it some architectural thing, or for a CPU bug, or just
over-caution maybe? They're not needed for correctness.
- Have unlikely conditional branches go forwards to help the static branch
predictor.
Remove memory barriers from the atomic ops macros in the same way as was
done for the other atomic ops earlier.
Use the correct barriers - all of membar_{sync,producer,consumer} have
less scope than before.
LGTM from riastradh
As we're providing the legacy gcc __sync built-in functions for atomic
memory access we might as well get the memory barriers right...
From the gcc documentation:
In most cases, these built-in functions are considered a full barrier.
That is, no memory operand is moved across the operation, either forward
or backward. Further, instructions are issued as necessary to prevent the
processor from speculating loads across the operation and from queuing
stores after the operation.
type __sync_lock_test_and_set (type *ptr, type value, ...)
This built-in function is not a full barrier, but rather an acquire
barrier. This means that references after the operation cannot move to
(or be speculated to) before the operation, but previous memory stores
may not be globally visible yet, and previous memory loads may not yet
be satisfied.
void __sync_lock_release (type *ptr, ...)
This built-in function is not a full barrier, but rather a release
barrier. This means that all previous memory stores are globally
visible, and all previous memory loads have been satisfied, but
following memory reads are not prevented from being speculated to
before the barrier.
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.1.28.1 \
src/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S \
src/common/lib/libc/arch/aarch64/atomic/membar_ops.S
cvs rdiff -u -r1.3 -r1.3.2.1 \
src/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h
cvs rdiff -u -r1.1.28.2 -r1.1.28.3 \
src/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S
cvs rdiff -u -r1.1.28.1 -r1.1.28.2 \
src/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S \
src/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP16(add, add)
+SYNC_FETCH_OP16(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_16,_atomic_add_16)
ATOMIC_OP_ALIAS(atomic_add_short,_atomic_add_16)
-STRONG_ALIAS(__sync_fetch_and_add_2,_atomic_add_16)
STRONG_ALIAS(_atomic_add_short,_atomic_add_16)
ATOMIC_OP16_NV(add, add)
+SYNC_OP16_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_16_nv,_atomic_add_16_nv)
ATOMIC_OP_ALIAS(atomic_add_short_nv,_atomic_add_16_nv)
-STRONG_ALIAS(__sync_add_and_fetch_2,_atomic_add_16_nv)
STRONG_ALIAS(_atomic_add_short_nv,_atomic_add_16_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(add, add)
+SYNC_FETCH_OP32(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_32,_atomic_add_32)
ATOMIC_OP_ALIAS(atomic_add_int,_atomic_add_32)
-STRONG_ALIAS(__sync_fetch_and_add_4,_atomic_add_32)
STRONG_ALIAS(_atomic_add_int,_atomic_add_32)
ATOMIC_OP32_NV(add, add)
+SYNC_OP32_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_32_nv,_atomic_add_32_nv)
ATOMIC_OP_ALIAS(atomic_add_int_nv,_atomic_add_32_nv)
-STRONG_ALIAS(__sync_add_and_fetch_4,_atomic_add_32_nv)
STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,18 +33,20 @@
ATOMIC_OP64(add, add)
+SYNC_FETCH_OP64(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_64_nv,_atomic_add_64_nv)
ATOMIC_OP_ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv)
ATOMIC_OP_ALIAS(atomic_add_long_nv,_atomic_add_64_nv)
STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv)
STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv)
-STRONG_ALIAS(__sync_add_and_fetch_8,_atomic_add_64_nv)
ATOMIC_OP64_NV(add, add)
+SYNC_OP64_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_64,_atomic_add_64)
ATOMIC_OP_ALIAS(atomic_add_ptr,_atomic_add_64)
ATOMIC_OP_ALIAS(atomic_add_long,_atomic_add_64)
STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64)
STRONG_ALIAS(_atomic_add_long,_atomic_add_64)
-STRONG_ALIAS(__sync_fetch_and_add_8,_atomic_add_64)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(add, add)
+SYNC_FETCH_OP8(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_8,_atomic_add_8)
ATOMIC_OP_ALIAS(atomic_add_char,_atomic_add_8)
-STRONG_ALIAS(__sync_fetch_and_add_1,_atomic_add_8)
STRONG_ALIAS(_atomic_add_char,_atomic_add_8)
ATOMIC_OP8_NV(add, add)
+SYNC_OP8_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_8_nv,_atomic_add_8_nv)
ATOMIC_OP_ALIAS(atomic_add_char_nv,_atomic_add_8_nv)
-STRONG_ALIAS(__sync_add_and_fetch_1,_atomic_add_8_nv)
STRONG_ALIAS(_atomic_add_char_nv,_atomic_add_8_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP16(and, and)
+SYNC_FETCH_OP16(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_16,_atomic_and_16)
ATOMIC_OP_ALIAS(atomic_and_ushort,_atomic_and_16)
-STRONG_ALIAS(__sync_fetch_and_and_2,_atomic_and_16)
STRONG_ALIAS(_atomic_and_ushort,_atomic_and_16)
ATOMIC_OP16_NV(and, and)
+SYNC_OP16_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_16_nv,_atomic_and_16_nv)
ATOMIC_OP_ALIAS(atomic_and_ushort_nv,_atomic_and_16_nv)
-STRONG_ALIAS(__sync_and_and_fetch_2,_atomic_and_16_nv)
STRONG_ALIAS(_atomic_and_ushort_nv,_atomic_and_16_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(and, and)
+SYNC_FETCH_OP32(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_32,_atomic_and_32)
ATOMIC_OP_ALIAS(atomic_and_uint,_atomic_and_32)
-STRONG_ALIAS(__sync_fetch_and_and_4,_atomic_and_32)
STRONG_ALIAS(_atomic_and_uint,_atomic_and_32)
ATOMIC_OP32_NV(and, and)
+SYNC_OP32_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_32_nv,_atomic_and_32_nv)
ATOMIC_OP_ALIAS(atomic_and_uint_nv,_atomic_and_32_nv)
-STRONG_ALIAS(__sync_and_and_fetch_4,_atomic_and_32_nv)
STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP64(and, and)
+SYNC_FETCH_OP64(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_64,_atomic_and_64)
ATOMIC_OP_ALIAS(atomic_and_ulong,_atomic_and_64)
STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64)
-STRONG_ALIAS(__sync_fetch_and_and_8,_atomic_and_64)
ATOMIC_OP64_NV(and, and)
+SYNC_OP64_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_64_nv,_atomic_and_64_nv)
ATOMIC_OP_ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv)
STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv)
-STRONG_ALIAS(__sync_and_and_fetch_8,_atomic_and_64_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(and, and)
+SYNC_FETCH_OP8(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_8,_atomic_and_8)
ATOMIC_OP_ALIAS(atomic_and_uchar,_atomic_and_8)
-STRONG_ALIAS(__sync_fetch_and_and_1,_atomic_and_8)
STRONG_ALIAS(_atomic_and_uchar,_atomic_and_8)
ATOMIC_OP8_NV(and, and)
+SYNC_OP8_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_8_nv,_atomic_and_8_nv)
ATOMIC_OP_ALIAS(atomic_and_uchar_nv,_atomic_and_8_nv)
-STRONG_ALIAS(__sync_and_and_fetch_1,_atomic_and_8_nv)
STRONG_ALIAS(_atomic_and_uchar_nv,_atomic_and_8_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,12 +37,25 @@ ENTRY_NP(_atomic_cas_16)
cmp w0, w1 /* compare? */
b.ne 2f
stxrh w3, w2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st /* data memory barrier */
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_16)
ATOMIC_OP_ALIAS(atomic_cas_16,_atomic_cas_16)
STRONG_ALIAS(_atomic_cas_short,_atomic_cas_16)
STRONG_ALIAS(_atomic_cas_ushort,_atomic_cas_16)
STRONG_ALIAS(__sync_val_compare_and_swap_2,_atomic_cas_16)
+
+ENTRY_NP(__sync_val_compare_and_swap_2)
+ mov x4, x0 /* we need x0 for return value */
+ dmb ish
+1: ldxrh w0, [x4] /* load old value */
+ cmp w0, w1 /* compare? */
+ b.ne 2f
+ stxrh w3, w2, [x4] /* store new value */
+ cbnz w3, 3f /* succeed? nope, try again. */
+ dmb ish
+2: ret /* return. */
+3: b 1b
+END(__sync_val_compare_and_swap_2)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,16 +37,28 @@ ENTRY_NP(_atomic_cas_32)
cmp w0, w1 /* compare? */
b.ne 2f /* return if different */
stxr w3, w2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_32_ni,_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_uint_ni,_atomic_cas_32)
-STRONG_ALIAS(__sync_val_compare_and_swap_4,_atomic_cas_32)
STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32)
STRONG_ALIAS(_atomic_cas_32_ni,_atomic_cas_32)
STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32)
+
+ENTRY_NP(__sync_val_compare_and_swap_4)
+ mov x4, x0 /* we need x0 for return value */
+ dmb ish
+1: ldxr w0, [x4] /* load old value */
+ cmp w0, w1 /* compare? */
+ b.ne 2f /* return if different */
+ stxr w3, w2, [x4] /* store new value */
+ cbnz w3, 3f /* succeed? nope, try again. */
+ dmb ish
+2: ret /* return. */
+3: b 1b
+END(__sync_val_compare_and_swap_4)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_cas_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,12 +37,24 @@ ENTRY_NP(_atomic_cas_8)
cmp w0, w1 /* compare? */
b.ne 2f
stxrb w3, w2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st /* data memory barrier */
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_8)
ATOMIC_OP_ALIAS(atomic_cas_8,_atomic_cas_8)
STRONG_ALIAS(_atomic_cas_char,_atomic_cas_8)
STRONG_ALIAS(_atomic_cas_uchar,_atomic_cas_8)
-STRONG_ALIAS(__sync_val_compare_and_swap_1,_atomic_cas_8)
+
+ENTRY_NP(__sync_val_compare_and_swap_1)
+ mov x4, x0 /* we need x0 for return value */
+ dmb ish
+1: ldxrb w0, [x4] /* load old value */
+ cmp w0, w1 /* compare? */
+ b.ne 2f
+ stxrb w3, w2, [x4] /* store new value */
+ cbnz w3, 3f /* succeed? nope, try again. */
+ dmb ish
+2: ret /* return. */
+3: b 1b
+END(__sync_val_compare_and_swap_1)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_dec_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@ ENTRY_NP(_atomic_dec_32)
1: ldxr w3, [x0] /* load old value (return value) */
sub w3, w3, #1 /* calculate new value */
stxr w2, w3, [x0] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_32)
ATOMIC_OP_ALIAS(atomic_dec_32,_atomic_dec_32)
@@ -49,9 +49,9 @@ ENTRY_NP(_atomic_dec_32_nv)
1: ldxr w0, [x4] /* load old value */
sub w0, w0, #1 /* calculate new value (return value) */
stxr w3, w0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_32_nv)
ATOMIC_OP_ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_dec_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@ ENTRY_NP(_atomic_dec_64)
1: ldxr x2, [x0] /* load old value (return value) */
sub x2, x2, #1 /* calculate new value */
stxr w3, x2, [x0] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_64)
ATOMIC_OP_ALIAS(atomic_dec_64,_atomic_dec_64)
@@ -51,9 +51,9 @@ ENTRY_NP(_atomic_dec_64_nv)
1: ldxr x0, [x4] /* load old value */
sub x0, x0, #1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_dec_64_nv)
ATOMIC_OP_ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_inc_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@ ENTRY_NP(_atomic_inc_32)
1: ldxr w3, [x0] /* load old value (return value) */
add w3, w3, #1 /* calculate new value */
stxr w2, w3, [x0] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_32)
ATOMIC_OP_ALIAS(atomic_inc_32,_atomic_inc_32)
@@ -49,9 +49,9 @@ ENTRY_NP(_atomic_inc_32_nv)
1: ldxr w0, [x4] /* load old value */
add w0, w0, #1 /* calculate new value (return value) */
stxr w3, w0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_32_nv)
ATOMIC_OP_ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_inc_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,9 +35,9 @@ ENTRY_NP(_atomic_inc_64)
1: ldxr x2, [x0] /* load old value (return value) */
add x2, x2, #1 /* calculate new value */
stxr w3, x2, [x0] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_64)
ATOMIC_OP_ALIAS(atomic_inc_64,_atomic_inc_64)
@@ -51,9 +51,9 @@ ENTRY_NP(_atomic_inc_64_nv)
1: ldxr x0, [x4] /* load old value */
add x0, x0, #1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_inc_64_nv)
ATOMIC_OP_ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_or_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_or_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP16(or, orr)
+SYNC_FETCH_OP16(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_16,_atomic_or_16)
ATOMIC_OP_ALIAS(atomic_or_ushort,_atomic_or_16)
-STRONG_ALIAS(__sync_fetch_and_or_2,_atomic_or_16)
STRONG_ALIAS(_atomic_or_ushort,_atomic_or_16)
ATOMIC_OP16_NV(or, orr)
+SYNC_OP16_FETCH(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_16_nv,_atomic_or_16_nv)
ATOMIC_OP_ALIAS(atomic_or_ushort_nv,_atomic_or_16_nv)
-STRONG_ALIAS(__sync_or_and_fetch_2,_atomic_or_16_nv)
STRONG_ALIAS(_atomic_or_ushort_nv,_atomic_or_16_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_or_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_or_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(or, orr)
+SYNC_FETCH_OP32(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_32,_atomic_or_32)
ATOMIC_OP_ALIAS(atomic_or_uint,_atomic_or_32)
-STRONG_ALIAS(__sync_fetch_and_or_4,_atomic_or_32)
STRONG_ALIAS(_atomic_or_uint,_atomic_or_32)
ATOMIC_OP32_NV(or, orr)
+SYNC_OP32_FETCH(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_32_nv,_atomic_or_32_nv)
ATOMIC_OP_ALIAS(atomic_or_uint_nv,_atomic_or_32_nv)
-STRONG_ALIAS(__sync_or_and_fetch_4,_atomic_or_32_nv)
STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_or_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_or_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(or, orr)
+SYNC_FETCH_OP8(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_8,_atomic_or_8)
ATOMIC_OP_ALIAS(atomic_or_uchar,_atomic_or_8)
-STRONG_ALIAS(__sync_fetch_and_or_1,_atomic_or_8)
STRONG_ALIAS(_atomic_or_uchar,_atomic_or_8)
ATOMIC_OP8_NV(or, orr)
+SYNC_OP8_FETCH(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_8_nv,_atomic_or_8_nv)
ATOMIC_OP_ALIAS(atomic_or_uchar_nv,_atomic_or_8_nv)
-STRONG_ALIAS(__sync_or_and_fetch_1,_atomic_or_8_nv)
STRONG_ALIAS(_atomic_or_uchar_nv,_atomic_or_8_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_sub_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_sub_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,18 +33,20 @@
ATOMIC_OP16(sub, sub)
+SYNC_FETCH_OP16(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_16,_atomic_sub_16)
ATOMIC_OP_ALIAS(atomic_sub_short,_atomic_sub_16)
ATOMIC_OP_ALIAS(atomic_sub_ushort,_atomic_sub_16)
-STRONG_ALIAS(__sync_fetch_and_sub_2,_atomic_sub_16)
STRONG_ALIAS(_atomic_sub_short,_atomic_sub_16)
STRONG_ALIAS(_atomic_sub_ushort,_atomic_sub_16)
ATOMIC_OP16_NV(sub, sub)
+SYNC_OP16_FETCH(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_16_nv,_atomic_sub_16_nv)
ATOMIC_OP_ALIAS(atomic_sub_short_nv,_atomic_sub_16_nv)
ATOMIC_OP_ALIAS(atomic_sub_ushort_nv,_atomic_sub_16_nv)
-STRONG_ALIAS(__sync_sub_and_fetch_2,_atomic_sub_16_nv)
STRONG_ALIAS(_atomic_sub_short_nv,_atomic_sub_16_nv)
STRONG_ALIAS(_atomic_sub_ushort_nv,_atomic_sub_16_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_sub_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_sub_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(sub, sub)
+SYNC_FETCH_OP32(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_32,_atomic_sub_32)
ATOMIC_OP_ALIAS(atomic_sub_int,_atomic_sub_32)
-STRONG_ALIAS(__sync_fetch_and_sub_4,_atomic_sub_32)
STRONG_ALIAS(_atomic_sub_int,_atomic_sub_32)
ATOMIC_OP32_NV(sub, sub)
+SYNC_OP32_FETCH(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_32_nv,_atomic_sub_32_nv)
ATOMIC_OP_ALIAS(atomic_sub_int_nv,_atomic_sub_32_nv)
-STRONG_ALIAS(__sync_sub_and_fetch_4,_atomic_sub_32_nv)
STRONG_ALIAS(_atomic_sub_int_nv,_atomic_sub_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_sub_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_sub_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,18 +33,20 @@
ATOMIC_OP64(sub, sub)
+SYNC_FETCH_OP64(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_64_nv,_atomic_sub_64_nv)
ATOMIC_OP_ALIAS(atomic_sub_ptr_nv,_atomic_sub_64_nv)
ATOMIC_OP_ALIAS(atomic_sub_long_nv,_atomic_sub_64_nv)
STRONG_ALIAS(_atomic_sub_ptr_nv,_atomic_sub_64_nv)
STRONG_ALIAS(_atomic_sub_long_nv,_atomic_sub_64_nv)
-STRONG_ALIAS(__sync_sub_and_fetch_8,_atomic_sub_64_nv)
ATOMIC_OP64_NV(sub, sub)
+SYNC_OP64_FETCH(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_64,_atomic_sub_64)
ATOMIC_OP_ALIAS(atomic_sub_ptr,_atomic_sub_64)
ATOMIC_OP_ALIAS(atomic_sub_long,_atomic_sub_64)
STRONG_ALIAS(_atomic_sub_ptr,_atomic_sub_64)
STRONG_ALIAS(_atomic_sub_long,_atomic_sub_64)
-STRONG_ALIAS(__sync_fetch_and_sub_8,_atomic_sub_64)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_sub_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_sub_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(sub, sub)
+SYNC_FETCH_OP8(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_8,_atomic_sub_8)
ATOMIC_OP_ALIAS(atomic_sub_char,_atomic_sub_8)
-STRONG_ALIAS(__sync_fetch_and_sub_1,_atomic_sub_8)
STRONG_ALIAS(_atomic_sub_char,_atomic_sub_8)
ATOMIC_OP8_NV(sub, sub)
+SYNC_OP8_FETCH(sub, sub)
+
ATOMIC_OP_ALIAS(atomic_sub_8_nv,_atomic_sub_8_nv)
ATOMIC_OP_ALIAS(atomic_sub_char_nv,_atomic_sub_8_nv)
-STRONG_ALIAS(__sync_sub_and_fetch_1,_atomic_sub_8_nv)
STRONG_ALIAS(_atomic_sub_char_nv,_atomic_sub_8_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_swap_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_swap_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,18 +35,26 @@ ENTRY_NP(_atomic_swap_16)
mov x4, x0
1: ldxrh w0, [x4]
stxrh w3, w1, [x4]
- cbnz w3, 1b
- dmb st
+ cbnz w3, 2f
ret
+2: b 1b
END(_atomic_swap_16)
ATOMIC_OP_ALIAS(atomic_swap_16,_atomic_swap_16)
ATOMIC_OP_ALIAS(atomic_swap_short,_atomic_swap_16)
ATOMIC_OP_ALIAS(atomic_swap_ushort,_atomic_swap_16)
-STRONG_ALIAS(__sync_lock_test_and_set_2,_atomic_swap_16)
STRONG_ALIAS(_atomic_swap_short,_atomic_swap_16)
STRONG_ALIAS(_atomic_swap_ushort,_atomic_swap_16)
+ENTRY_NP(__sync_lock_test_and_set_2)
+ mov x4, x0
+1: ldaxrh w0, [x4]
+ stxrh w3, w1, [x4]
+ cbnz w3, 2f
+ ret
+2: b 1b
+END(__sync_lock_test_and_set_2)
+
ENTRY_NP(__sync_lock_release_2)
stlrh wzr, [x0]
ret
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_swap_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_swap_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,16 +35,24 @@ ENTRY_NP(_atomic_swap_32)
mov x4, x0
1: ldxr w0, [x4]
stxr w3, w1, [x4]
- cbnz w3, 1b
- dmb st
+ cbnz w3, 2f
ret
+2: b 1b
END(_atomic_swap_32)
ATOMIC_OP_ALIAS(atomic_swap_32,_atomic_swap_32)
ATOMIC_OP_ALIAS(atomic_swap_uint,_atomic_swap_32)
-STRONG_ALIAS(__sync_lock_test_and_set_4,_atomic_swap_32)
STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32)
+ENTRY_NP(__sync_lock_test_and_set_4)
+ mov x4, x0
+1: ldaxr w0, [x4]
+ stxr w3, w1, [x4]
+ cbnz w3, 2f
+ ret
+2: b 1b
+END(__sync_lock_test_and_set_4)
+
ENTRY_NP(__sync_lock_release_4)
stlr wzr, [x0]
ret
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_swap_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_swap_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,18 +35,26 @@ ENTRY_NP(_atomic_swap_64)
mov x4, x0
1: ldxr x0, [x4]
stxr w3, x1, [x4]
- cbnz w3, 1b
- dmb st
+ cbnz w3, 2f
ret
+2: b 1b
END(_atomic_swap_64)
ATOMIC_OP_ALIAS(atomic_swap_64,_atomic_swap_64)
ATOMIC_OP_ALIAS(atomic_swap_ulong,_atomic_swap_64)
ATOMIC_OP_ALIAS(atomic_swap_ptr,_atomic_swap_64)
-STRONG_ALIAS(__sync_lock_test_and_set_8,_atomic_swap_64)
STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64)
STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64)
+ENTRY_NP(__sync_lock_test_and_set_8)
+ mov x4, x0
+1: ldaxr x0, [x4]
+ stxr w3, x1, [x4]
+ cbnz w3, 2f
+ ret
+2: b 1b
+END(_atomic_swap_64)
+
ENTRY_NP(__sync_lock_release_8)
stlr xzr, [x0]
ret
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_swap_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_swap_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,18 +35,26 @@ ENTRY_NP(_atomic_swap_8)
mov x4, x0
1: ldxrb w0, [x4]
stxrb w3, w1, [x4]
- cbnz w3, 1b
- dmb st
+ cbnz w3, 2f
ret
+2: b 1b
END(_atomic_swap_8)
ATOMIC_OP_ALIAS(atomic_swap_8,_atomic_swap_8)
ATOMIC_OP_ALIAS(atomic_swap_char,_atomic_swap_8)
ATOMIC_OP_ALIAS(atomic_swap_uchar,_atomic_swap_8)
-STRONG_ALIAS(__sync_lock_test_and_set_1,_atomic_swap_8)
STRONG_ALIAS(_atomic_swap_char,_atomic_swap_8)
STRONG_ALIAS(_atomic_swap_uchar,_atomic_swap_8)
+ENTRY_NP(__sync_lock_test_and_set_1)
+ mov x4, x0
+1: ldaxrb w0, [x4]
+ stxrb w3, w1, [x4]
+ cbnz w3, 2f
+ ret
+2: b 1b
+END(__sync_lock_test_and_set_1)
+
ENTRY_NP(__sync_lock_release_1)
stlrb wzr, [x0]
ret
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_xor_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_xor_16.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP16(xor, eor)
+SYNC_FETCH_OP16(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_16,_atomic_xor_16)
ATOMIC_OP_ALIAS(atomic_xor_ushort,_atomic_xor_16)
-STRONG_ALIAS(__sync_fetch_and_xor_2,_atomic_xor_16)
STRONG_ALIAS(_atomic_xor_ushort,_atomic_xor_16)
ATOMIC_OP16_NV(xor, eor)
+SYNC_OP16_FETCH(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_16_nv,_atomic_xor_16_nv)
ATOMIC_OP_ALIAS(atomic_xor_ushort_nv,_atomic_xor_16_nv)
-STRONG_ALIAS(__sync_xor_and_fetch_2,_atomic_xor_16_nv)
STRONG_ALIAS(_atomic_xor_ushort_nv,_atomic_xor_16_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_xor_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_xor_32.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(xor, eor)
+SYNC_FETCH_OP32(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_32,_atomic_xor_32)
ATOMIC_OP_ALIAS(atomic_xor_uint,_atomic_xor_32)
-STRONG_ALIAS(__sync_fetch_and_xor_4,_atomic_xor_32)
STRONG_ALIAS(_atomic_xor_uint,_atomic_xor_32)
ATOMIC_OP32_NV(xor, eor)
+SYNC_OP32_FETCH(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_32_nv,_atomic_xor_32_nv)
ATOMIC_OP_ALIAS(atomic_xor_uint_nv,_atomic_xor_32_nv)
-STRONG_ALIAS(__sync_xor_and_fetch_4,_atomic_xor_32_nv)
STRONG_ALIAS(_atomic_xor_uint_nv,_atomic_xor_32_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_xor_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_xor_64.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,12 +33,14 @@
ATOMIC_OP64(xor, eor)
+SYNC_FETCH_OP64(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_64,_atomic_xor_64)
ATOMIC_OP_ALIAS(atomic_xor_ulong,_atomic_xor_64)
-STRONG_ALIAS(__sync_fetch_and_xor_8,_atomic_xor_64)
ATOMIC_OP64_NV(xor, eor)
+SYNC_OP64_FETCH(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_64_nv,_atomic_xor_64_nv)
ATOMIC_OP_ALIAS(atomic_xor_ulong_nv,_atomic_xor_64_nv)
-STRONG_ALIAS(__sync_xor_and_fetch_8,_atomic_xor_64_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S:1.1 src/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_xor_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_xor_8.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(xor, eor)
+SYNC_FETCH_OP8(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_8,_atomic_xor_8)
ATOMIC_OP_ALIAS(atomic_xor_uchar,_atomic_xor_8)
-STRONG_ALIAS(__sync_fetch_and_xor_1,_atomic_xor_8)
STRONG_ALIAS(_atomic_xor_uchar,_atomic_xor_8)
ATOMIC_OP8_NV(xor, eor)
+SYNC_OP8_FETCH(xor, eor)
+
ATOMIC_OP_ALIAS(atomic_xor_8_nv,_atomic_xor_8_nv)
ATOMIC_OP_ALIAS(atomic_xor_uchar_nv,_atomic_xor_8_nv)
-STRONG_ALIAS(__sync_xor_and_fetch_1,_atomic_xor_8_nv)
STRONG_ALIAS(_atomic_xor_uchar_nv,_atomic_xor_8_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/membar_ops.S
diff -u src/common/lib/libc/arch/aarch64/atomic/membar_ops.S:1.1 src/common/lib/libc/arch/aarch64/atomic/membar_ops.S:1.1.28.1
--- src/common/lib/libc/arch/aarch64/atomic/membar_ops.S:1.1 Sun Aug 10 05:47:35 2014
+++ src/common/lib/libc/arch/aarch64/atomic/membar_ops.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: membar_ops.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: membar_ops.S,v 1.1.28.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -32,24 +32,28 @@
#include "atomic_op_asm.h"
ENTRY_NP(_membar_producer)
- dsb sy
+ dmb ishst
ret
END(_membar_producer)
ATOMIC_OP_ALIAS(membar_producer,_membar_producer)
ATOMIC_OP_ALIAS(membar_write,_membar_producer)
STRONG_ALIAS(_membar_write,_membar_producer)
+ENTRY_NP(_membar_consumer)
+ dmb ishld
+ ret
+END(_membar_producer)
+ATOMIC_OP_ALIAS(membar_consumer,_membar_consumer)
+ATOMIC_OP_ALIAS(membar_read,_membar_consumer)
+STRONG_ALIAS(_membar_read,_membar_consumer)
+
ENTRY_NP(_membar_sync)
- dmb sy
+ dmb ish
ret
END(_membar_sync)
ATOMIC_OP_ALIAS(membar_sync,_membar_sync)
ATOMIC_OP_ALIAS(membar_enter,_membar_sync)
ATOMIC_OP_ALIAS(membar_exit,_membar_sync)
-ATOMIC_OP_ALIAS(membar_consumer,_membar_sync)
-ATOMIC_OP_ALIAS(membar_read,_membar_sync)
STRONG_ALIAS(__sync_synchronize,_membar_sync)
STRONG_ALIAS(_membar_enter,_membar_sync)
STRONG_ALIAS(_membar_exit,_membar_sync)
-STRONG_ALIAS(_membar_consumer,_membar_sync)
-STRONG_ALIAS(_membar_read,_membar_sync)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S:1.3 src/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S:1.3.2.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S:1.3 Tue Feb 19 12:51:44 2019
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_64.S,v 1.3 2019/02/19 12:51:44 rin Exp $ */
+/* $NetBSD: atomic_cas_64.S,v 1.3.2.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -37,9 +37,9 @@ ENTRY_NP(_atomic_cas_64)
cmp x0, x1 /* compare? */
b.ne 2f /* return if different */
stxr w3, x2, [x4] /* store new value */
- cbnz w3, 1b /* succeed? nope, try again. */
- dmb st
+ cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
+3: b 1b
END(_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_64,_atomic_cas_64)
@@ -48,9 +48,21 @@ ATOMIC_OP_ALIAS(atomic_cas_ptr,_atomic_c
ATOMIC_OP_ALIAS(atomic_cas_64_ni,_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_ulong_ni,_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_ptr_ni,_atomic_cas_64)
-STRONG_ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_64_ni,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64)
+
+ENTRY_NP(__sync_val_compare_and_swap_8)
+ mov x4, x0 /* we need x0 for return value */
+ dmb ish
+1: ldxr x0, [x4] /* load old value */
+ cmp x0, x1 /* compare? */
+ b.ne 2f /* return if different */
+ stxr w3, x2, [x4] /* store new value */
+ cbnz w3, 3f /* succeed? nope, try again. */
+ dmb ish
+2: ret /* return. */
+3: b 1b
+END(__sync_val_compare_and_swap_8)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h:1.3 src/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h:1.3.2.1
--- src/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h:1.3 Fri Feb 8 06:56:56 2019
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_op_asm.h,v 1.3 2019/02/08 06:56:56 ryo Exp $ */
+/* $NetBSD: atomic_op_asm.h,v 1.3.2.1 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -41,10 +41,21 @@ ENTRY_NP(_atomic_##OP##_8) ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxrb w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
- dmb st ;\
ret /* return old value */ ;\
END(_atomic_##OP##_8)
+#define SYNC_FETCH_OP8(OP, INSN) \
+ENTRY_NP(__sync_fetch_and_##OP##_1) ;\
+ mov x4, x0 ;\
+ dmb ish ;\
+1: ldxrb w0, [x4] /* load old value */ ;\
+ INSN w2, w0, w1 /* calculate new value */ ;\
+ stxrb w3, w2, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again */ ;\
+ dmb ish ;\
+ ret /* return old value */ ;\
+END(__sync_fetch_and_##OP##_1)
+
#define ATOMIC_OP8_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_8_nv) ;\
mov x4, x0 /* need r0 for return value */ ;\
@@ -52,10 +63,21 @@ ENTRY_NP(_atomic_##OP##_8_nv) ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxrb w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
- dmb sy ;\
ret /* return new value */ ;\
END(_atomic_##OP##_8_nv)
+#define SYNC_OP8_FETCH(OP, INSN) \
+ENTRY_NP(__sync_##OP##_and_fetch_1) ;\
+ mov x4, x0 /* need x0 for return value */ ;\
+ dmb ish ;\
+1: ldxrb w0, [x4] /* load old value */ ;\
+ INSN w0, w0, w1 /* calc new (return) value */ ;\
+ stxrb w3, w0, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again */ ;\
+ dmb ish ;\
+ ret /* return new value */ ;\
+END(__sync_##OP##_and_fetch_1)
+
#define ATOMIC_OP16(OP, INSN) \
ENTRY_NP(_atomic_##OP##_16) ;\
mov x4, x0 ;\
@@ -63,10 +85,21 @@ ENTRY_NP(_atomic_##OP##_16) ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxrh w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
- dmb st ;\
ret /* return old value */ ;\
END(_atomic_##OP##_16)
+#define SYNC_FETCH_OP16(OP, INSN) \
+ENTRY_NP(__sync_fetch_and_##OP##_2) ;\
+ mov x4, x0 ;\
+ dmb ish ;\
+1: ldxrh w0, [x4] /* load old value */ ;\
+ INSN w2, w0, w1 /* calculate new value */ ;\
+ stxrh w3, w2, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again */ ;\
+ dmb ish ;\
+ ret /* return old value */ ;\
+END(__sync_fetch_and_##OP##_2)
+
#define ATOMIC_OP16_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_16_nv) ;\
mov x4, x0 /* need r0 for return value */ ;\
@@ -74,10 +107,21 @@ ENTRY_NP(_atomic_##OP##_16_nv) ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxrh w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
- dmb sy ;\
ret /* return new value */ ;\
END(_atomic_##OP##_16_nv)
+#define SYNC_OP16_FETCH(OP, INSN) \
+ENTRY_NP(__sync__##OP##_and_fetch_2) ;\
+ mov x4, x0 /* need x0 for return value */ ;\
+ dmb ish ;\
+1: ldxrh w0, [x4] /* load old value */ ;\
+ INSN w0, w0, w1 /* calc new (return) value */ ;\
+ stxrh w3, w0, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again */ ;\
+ dmb ish ;\
+ ret /* return new value */ ;\
+END(__sync__##OP##_and_fetch_2)
+
#define ATOMIC_OP32(OP, INSN) \
ENTRY_NP(_atomic_##OP##_32) ;\
mov x4, x0 ;\
@@ -85,10 +129,21 @@ ENTRY_NP(_atomic_##OP##_32) ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxr w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
- dmb st ;\
ret /* return old value */ ;\
END(_atomic_##OP##_32)
+#define SYNC_FETCH_OP32(OP, INSN) \
+ENTRY_NP(__sync_fetch_and_##OP##_4) ;\
+ mov x4, x0 ;\
+ dmb ish ;\
+1: ldxr w0, [x4] /* load old value */ ;\
+ INSN w2, w0, w1 /* calculate new value */ ;\
+ stxr w3, w2, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again */ ;\
+ dmb ish ;\
+ ret /* return old value */ ;\
+END(__sync_fetch_and_##OP##_4)
+
#define ATOMIC_OP32_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_32_nv) ;\
mov x4, x0 /* need r0 for return value */ ;\
@@ -96,10 +151,21 @@ ENTRY_NP(_atomic_##OP##_32_nv) ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxr w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again? */ ;\
- dmb sy ;\
ret /* return new value */ ;\
END(_atomic_##OP##_32_nv)
+#define SYNC_OP32_FETCH(OP, INSN) \
+ENTRY_NP(__sync__##OP##_and_fetch_4) ;\
+ mov x4, x0 /* need x0 for return value */ ;\
+ dmb ish ;\
+1: ldxr w0, [x4] /* load old value */ ;\
+ INSN w0, w0, w1 /* calc new (return) value */ ;\
+ stxr w3, w0, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again? */ ;\
+ dmb ish ;\
+ ret /* return new value */ ;\
+END(__sync__##OP##_and_fetch_4)
+
#define ATOMIC_OP64(OP, INSN) \
ENTRY_NP(_atomic_##OP##_64) ;\
mov x4, x0 ;\
@@ -107,10 +173,21 @@ ENTRY_NP(_atomic_##OP##_64) ;\
INSN x2, x0, x1 /* calculate new value */ ;\
stxr w3, x2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
- dmb st ;\
ret /* return old value */ ;\
END(_atomic_##OP##_64)
+#define SYNC_FETCH_OP64(OP, INSN) \
+ENTRY_NP(__sync_fetch_and_##OP##_8) ;\
+ mov x4, x0 ;\
+ dmb ish ;\
+1: ldxr x0, [x4] /* load old value */ ;\
+ INSN x2, x0, x1 /* calculate new value */ ;\
+ stxr w3, x2, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again */ ;\
+ dmb ish ;\
+ ret /* return old value */ ;\
+END(__sync_fetch_and_##OP##_8)
+
#define ATOMIC_OP64_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_64_nv) ;\
mov x4, x0 /* need r0 for return value */ ;\
@@ -118,10 +195,21 @@ ENTRY_NP(_atomic_##OP##_64_nv) ;\
INSN x0, x0, x1 /* calc new (return) value */ ;\
stxr w3, x0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again? */ ;\
- dmb sy ;\
ret /* return new value */ ;\
END(_atomic_##OP##_64_nv)
+#define SYNC_OP64_FETCH(OP, INSN) \
+ENTRY_NP(__sync_##OP##_and_fetch_8) ;\
+ mov x4, x0 /* need x0 for return value */ ;\
+ dmb ish ;\
+1: ldxr x0, [x4] /* load old value */ ;\
+ INSN x0, x0, x1 /* calc new (return) value */ ;\
+ stxr w3, x0, [x4] /* try to store */ ;\
+ cbnz w3, 1b /* succeed? no, try again? */ ;\
+ dmb ish ;\
+ ret /* return new value */ ;\
+END(__sync_##OP##_and_fetch_8)
+
#if defined(_KERNEL)
#define ATOMIC_OP_ALIAS(a,s) STRONG_ALIAS(a,s)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S:1.1.28.2 src/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S:1.1.28.3
--- src/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S:1.1.28.2 Thu Jul 8 11:22:01 2021
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_16.S,v 1.1.28.2 2021/07/08 11:22:01 martin Exp $ */
+/* $NetBSD: atomic_nand_16.S,v 1.1.28.3 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,19 +35,31 @@
* { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand
*/
ENTRY_NP(_atomic_nand_16)
- mov x4, x0
+ mov x4, x0 /* need r0 for return value */
1: ldxrh w0, [x4] /* load old value (*ptr) */
and w3, w0, w1 /* w3 = (*ptr & value) */
mvn w3, w3 /* w3 = ~(*ptr & value) */
stxrh w2, w3, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
END(_atomic_nand_16)
+ENTRY_NP(__sync_fetch_and_nand_2)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxrh w0, [x4] /* load old value (*ptr) */
+ and w3, w0, w1 /* w3 = (*ptr & value) */
+ mvn w3, w3 /* w3 = ~(*ptr & value) */
+ stxrh w2, w3, [x4] /* try to store */
+ cbnz w2, 2f /* succeed? no, try again */
+ dmb ish
+ ret /* return old value */
+2: b 1b
+END(__sync_fetch_and_nand_2)
+
ATOMIC_OP_ALIAS(atomic_nand_16,_atomic_nand_16)
ATOMIC_OP_ALIAS(atomic_nand_ushort,_atomic_nand_16)
-STRONG_ALIAS(__sync_fetch_and_nand_2,_atomic_nand_16)
STRONG_ALIAS(_atomic_nand_ushort,_atomic_nand_16)
@@ -60,12 +72,24 @@ ENTRY_NP(_atomic_nand_16_nv)
and w0, w0, w1 /* w0 = (*ptr & value) */
mvn w0, w0 /* w0 = ~(*ptr & value), return value */
stxrh w2, w0, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_nand_16_nv)
+ENTRY_NP(__sync_nand_and_fetch_2)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxrh w0, [x4] /* load old value (*ptr) */
+ and w0, w0, w1 /* w0 = (*ptr & value) */
+ mvn w0, w0 /* w0 = ~(*ptr & value), return */
+ stxrh w2, w0, [x4] /* try to store */
+ cbnz w2, 2f /* succeed? no, try again? */
+ dmb ish
+ ret /* return new value */
+2: b 1b
+END(__sync_nand_and_fetch_2)
+
ATOMIC_OP_ALIAS(atomic_nand_16_nv,_atomic_nand_16_nv)
ATOMIC_OP_ALIAS(atomic_nand_ushort_nv,_atomic_nand_16_nv)
-STRONG_ALIAS(__sync_nand_and_fetch_2,_atomic_nand_16_nv)
STRONG_ALIAS(_atomic_nand_ushort_nv,_atomic_nand_16_nv)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S:1.1.28.1 src/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S:1.1.28.2
--- src/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S:1.1.28.1 Tue Jul 6 04:13:50 2021
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_32.S,v 1.1.28.1 2021/07/06 04:13:50 martin Exp $ */
+/* $NetBSD: atomic_nand_32.S,v 1.1.28.2 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,21 +35,33 @@
* { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand
*/
ENTRY_NP(_atomic_nand_32)
- mov x4, x0
-1: ldxr w0, [x4] /* load old value (to be returned) */
+ mov x4, x0 /* need r0 for return value */
+1: ldxr w0, [x4] /* load old value (*ptr) */
and w3, w0, w1 /* w3 = (*ptr & value) */
- mvn w3, w3 /* x3 = ~(*ptr & value) */
+ mvn w3, w3 /* w3 = ~(*ptr & value) */
stxr w2, w3, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
END(_atomic_nand_32)
ATOMIC_OP_ALIAS(atomic_nand_32,_atomic_nand_32)
ATOMIC_OP_ALIAS(atomic_nand_uint,_atomic_nand_32)
-STRONG_ALIAS(__sync_fetch_and_nand_4,_atomic_nand_32)
STRONG_ALIAS(_atomic_nand_uint,_atomic_nand_32)
+ENTRY_NP(__sync_fetch_and_nand_4)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxr w0, [x4] /* load old value (*ptr) */
+ and w3, w0, w1 /* w3 = (*ptr & value) */
+ mvn w3, w3 /* w3 = ~(*ptr & value) */
+ stxr w2, w3, [x4] /* try to store */
+ cbnz w2, 2f /* succeed? no, try again */
+ dmb ish
+ ret /* return old value */
+2: b 1b
+END(__sync_fetch_and_nand_4)
+
/*
* { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand
@@ -57,15 +69,27 @@ STRONG_ALIAS(_atomic_nand_uint,_atomic_n
ENTRY_NP(_atomic_nand_32_nv)
mov x4, x0 /* need r0 for return value */
1: ldxr w0, [x4] /* load old value (*ptr) */
- and w0, w0, w1 /* x0 = (*ptr & value) */
- mvn w0, w0 /* x0 = ~(*ptr & value), return value */
+ and w0, w0, w1 /* w0 = (*ptr & value) */
+ mvn w0, w0 /* w0 = ~(*ptr & value), return value */
stxr w2, w0, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_nand_32_nv)
ATOMIC_OP_ALIAS(atomic_nand_32_nv,_atomic_nand_32_nv)
ATOMIC_OP_ALIAS(atomic_nand_uint_nv,_atomic_nand_32_nv)
-STRONG_ALIAS(__sync_nand_and_fetch_4,_atomic_nand_32_nv)
STRONG_ALIAS(_atomic_nand_uint_nv,_atomic_nand_32_nv)
+
+ENTRY_NP(__sync_nand_and_fetch_4)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxr w0, [x4] /* load old value (tmp) */
+ and w0, w0, w1 /* w0 = (tmp & value) */
+ mvn w0, w0 /* w0 = ~(tmp & value), return */
+ stxr w2, w0, [x4] /* try to store */
+ cbnz w2, 2f /* succeed? no, try again? */
+ dmb ish
+ ret /* return new value */
+2: b 1b
+END(__sync_nand_and_fetch_4)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S:1.1.28.1 src/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S:1.1.28.2
--- src/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S:1.1.28.1 Tue Jul 6 04:13:50 2021
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_64.S,v 1.1.28.1 2021/07/06 04:13:50 martin Exp $ */
+/* $NetBSD: atomic_nand_64.S,v 1.1.28.2 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,21 +35,33 @@
* { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand
*/
ENTRY_NP(_atomic_nand_64)
- mov x4, x0
+ mov x4, x0 /* need r0 for return value */
1: ldxr x0, [x4] /* load old value (*ptr) */
and x2, x0, x1 /* x2 = (*ptr & value) */
mvn x2, x2 /* x2 = ~(*ptr & value) */
stxr w3, x2, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
END(_atomic_nand_64)
ATOMIC_OP_ALIAS(atomic_nand_64,_atomic_nand_64)
ATOMIC_OP_ALIAS(atomic_nand_ulong,_atomic_nand_64)
-STRONG_ALIAS(__sync_fetch_and_nand_8,_atomic_nand_64)
STRONG_ALIAS(_atomic_nand_ulong,_atomic_nand_64)
+ENTRY_NP(__sync_fetch_and_nand_8)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxr x0, [x4] /* load old value (*ptr) */
+ and x2, x0, x1 /* x2 = (*ptr & value) */
+ mvn x2, x2 /* x2 = ~(*ptr & value) */
+ stxr w3, x2, [x4] /* try to store */
+ cbnz w3, 2f /* succeed? no, try again */
+ dmb ish
+ ret /* return old value */
+2: b 1b
+END(__sync_fetch_and_nand_8)
+
/*
* { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand
@@ -60,12 +72,24 @@ ENTRY_NP(_atomic_nand_64_nv)
and x0, x0, x1 /* x0 = (*ptr & value) */
mvn x0, x0 /* x0 = ~(*ptr & value), return value */
stxr w3, x0, [x4] /* try to store */
- cbnz w3, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_nand_64_nv)
ATOMIC_OP_ALIAS(atomic_nand_64_nv,_atomic_nand_64_nv)
ATOMIC_OP_ALIAS(atomic_nand_ulong_nv,_atomic_nand_64_nv)
-STRONG_ALIAS(__sync_nand_and_fetch_8,_atomic_nand_64_nv)
STRONG_ALIAS(_atomic_nand_ulong_nv,_atomic_nand_64_nv)
+
+ENTRY_NP(__sync_nand_and_fetch_8)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxr x0, [x4] /* load old value (*ptr) */
+ and x0, x0, x1 /* x0 = (*ptr & value) */
+ mvn x0, x0 /* x0 = ~(*ptr & value) */
+ stxr w3, x0, [x4] /* try to store */
+ cbnz w3, 2f /* succeed? no, try again? */
+ dmb ish
+ ret /* return new value */
+2: b 1b
+END(__sync_nand_and_fetch_8)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S:1.1.28.1 src/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S:1.1.28.2
--- src/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S:1.1.28.1 Tue Jul 6 04:13:50 2021
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_nand_8.S,v 1.1.28.1 2021/07/06 04:13:50 martin Exp $ */
+/* $NetBSD: atomic_nand_8.S,v 1.1.28.2 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -35,21 +35,34 @@
* { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand
*/
ENTRY_NP(_atomic_nand_8)
- mov x4, x0
+ mov x4, x0 /* need r0 for return value */
1: ldxrb w0, [x4] /* load old value (*ptr) */
and w3, w0, w1 /* w3 = (*ptr & value) */
mvn w3, w3 /* w3 = ~(*ptr & value) */
stxrb w2, w3, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
+2: b 1b
END(_atomic_nand_8)
ATOMIC_OP_ALIAS(atomic_nand_8,_atomic_nand_8)
ATOMIC_OP_ALIAS(atomic_nand_uchar,_atomic_nand_8)
-STRONG_ALIAS(__sync_fetch_and_nand_1,_atomic_nand_8)
STRONG_ALIAS(_atomic_nand_uchar,_atomic_nand_8)
+ENTRY_NP(__sync_fetch_and_nand_1)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxrb w0, [x4] /* load old value (*ptr) */
+ and w3, w0, w1 /* w3 = (*ptr & value) */
+ mvn w3, w3 /* w3 = ~(*ptr & value) */
+ stxrb w2, w3, [x4] /* try to store */
+ cbnz w2, 2f /* succeed? no, try again */
+ dmb ish
+ ret /* return old value */
+2: b 1b
+END(__sync_fetch_and_nand_1)
+
+
/*
* { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand
*/
@@ -59,12 +72,24 @@ ENTRY_NP(_atomic_nand_8_nv)
and w0, w0, w1 /* w0 = (*ptr & value) */
mvn w0, w0 /* w0 = ~(*ptr & value), return value */
stxrb w2, w0, [x4] /* try to store */
- cbnz w2, 1b /* succeed? no, try again? */
- dmb st
+ cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
+2: b 1b
END(_atomic_nand_8_nv)
ATOMIC_OP_ALIAS(atomic_nand_8_nv,_atomic_nand_8_nv)
ATOMIC_OP_ALIAS(atomic_nand_uchar_nv,_atomic_nand_8_nv)
-STRONG_ALIAS(__sync_nand_and_fetch_1,_atomic_nand_8_nv)
STRONG_ALIAS(_atomic_nand_uchar_nv,_atomic_nand_8_nv)
+
+ENTRY_NP(__sync_nand_and_fetch_1)
+ mov x4, x0 /* need r0 for return value */
+ dmb ish
+1: ldxrb w0, [x4] /* load old value (*ptr) */
+ and w0, w0, w1 /* w0 = (*ptr & value) */
+ mvn w0, w0 /* w0 = ~(*ptr & value) */
+ stxrb w2, w0, [x4] /* try to store */
+ cbnz w2, 2f /* succeed? no, try again? */
+ dmb ish
+ ret /* return new value */
+2: b 1b
+END(__sync_nand_and_fetch_1)
Index: src/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S
diff -u src/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S:1.1.28.1 src/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S:1.1.28.2
--- src/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S:1.1.28.1 Tue Sep 17 19:23:23 2019
+++ src/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S Wed Aug 11 17:19:01 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_or_64.S,v 1.1.28.1 2019/09/17 19:23:23 martin Exp $ */
+/* $NetBSD: atomic_or_64.S,v 1.1.28.2 2021/08/11 17:19:01 martin Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP64(or, orr)
+SYNC_FETCH_OP64(or, orr)
+
ATOMIC_OP_ALIAS(atomic_or_64,_atomic_or_64)
ATOMIC_OP_ALIAS(atomic_or_ulong,_atomic_or_64)
STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64)
-STRONG_ALIAS(__sync_fetch_and_or_8,_atomic_or_64)
ATOMIC_OP64_NV(or, orr)
+SYNC_OP64_FETCH(or, orr)
+
STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv)
ATOMIC_OP_ALIAS(atomic_or_64_nv,_atomic_or_64_nv)
ATOMIC_OP_ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv)
-STRONG_ALIAS(__sync_or_and_fetch_8,_atomic_or_64_nv)
