Module Name: src
Committed By: skrll
Date: Thu Aug 6 10:00:21 UTC 2020
Modified Files:
src/common/lib/libc/arch/mips/atomic: atomic_add.S atomic_and.S
atomic_cas.S atomic_dec.S atomic_inc.S atomic_or.S atomic_swap.S
src/sys/arch/mips/include: asm.h
src/sys/arch/mips/mips: lock_stubs_llsc.S
Log Message:
Centralise SYNC/BDSYNC in asm.h and introduce a new LLCSCSYNC and use it
before any ll/sc sequences.
Define LLSCSYNC as syncw; syncw for cnMIPS - issue two as early cnMIPS
has errat{um,a} that means the first can fail.
To generate a diff of this commit:
cvs rdiff -u -r1.6 -r1.7 src/common/lib/libc/arch/mips/atomic/atomic_add.S \
src/common/lib/libc/arch/mips/atomic/atomic_dec.S \
src/common/lib/libc/arch/mips/atomic/atomic_inc.S \
src/common/lib/libc/arch/mips/atomic/atomic_swap.S
cvs rdiff -u -r1.5 -r1.6 src/common/lib/libc/arch/mips/atomic/atomic_and.S \
src/common/lib/libc/arch/mips/atomic/atomic_or.S
cvs rdiff -u -r1.7 -r1.8 src/common/lib/libc/arch/mips/atomic/atomic_cas.S
cvs rdiff -u -r1.57 -r1.58 src/sys/arch/mips/include/asm.h
cvs rdiff -u -r1.10 -r1.11 src/sys/arch/mips/mips/lock_stubs_llsc.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/common/lib/libc/arch/mips/atomic/atomic_add.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_add.S:1.6 src/common/lib/libc/arch/mips/atomic/atomic_add.S:1.7
--- src/common/lib/libc/arch/mips/atomic/atomic_add.S:1.6 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_add.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add.S,v 1.6 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_add.S,v 1.7 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -29,7 +29,7 @@
#include <machine/asm.h>
#include "atomic_op_asm.h"
-RCSID("$NetBSD: atomic_add.S,v 1.6 2020/08/01 09:26:49 skrll Exp $")
+RCSID("$NetBSD: atomic_add.S,v 1.7 2020/08/06 10:00:21 skrll Exp $")
.text
.set noreorder
@@ -49,6 +49,7 @@ LEAF(_atomic_add_32)
#if defined(_MIPS_ARCH_OCTEONP) || defined(_MIPS_ARCH_OCTEON2)
saa a1, (a0)
#else
+ LLSCSYNC
1: INT_LL t0, 0(a0)
nop
INT_ADDU t0, a1
@@ -62,6 +63,7 @@ END(_atomic_add_32)
ATOMIC_OP_ALIAS(atomic_add_32, _atomic_add_32)
LEAF(_atomic_add_32_nv)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
INT_ADDU v0, a1
@@ -79,6 +81,7 @@ LEAF(_atomic_add_64)
#if defined(_MIPS_ARCH_OCTEONP) || defined(_MIPS_ARCH_OCTEON2)
saad a1, (a0)
#else
+ LLSCSYNC
1: REG_LL t0, 0(a0)
nop
REG_ADDU t0, a1
@@ -92,6 +95,7 @@ END(_atomic_add_64)
ATOMIC_OP_ALIAS(atomic_add_64, _atomic_add_64)
LEAF(_atomic_add_64_nv)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
REG_ADDU v0, a1
Index: src/common/lib/libc/arch/mips/atomic/atomic_dec.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_dec.S:1.6 src/common/lib/libc/arch/mips/atomic/atomic_dec.S:1.7
--- src/common/lib/libc/arch/mips/atomic/atomic_dec.S:1.6 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_dec.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_dec.S,v 1.6 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_dec.S,v 1.7 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -29,7 +29,7 @@
#include <machine/asm.h>
#include "atomic_op_asm.h"
-RCSID("$NetBSD: atomic_dec.S,v 1.6 2020/08/01 09:26:49 skrll Exp $")
+RCSID("$NetBSD: atomic_dec.S,v 1.7 2020/08/06 10:00:21 skrll Exp $")
.text
.set noreorder
@@ -49,6 +49,7 @@ LEAF(_atomic_dec_32)
li t0, -1
saa t0, (a0)
#else
+ LLSCSYNC
1: INT_LL t0, 0(a0)
nop
INT_ADDU t0, -1
@@ -62,6 +63,7 @@ END(_atomic_dec_32)
ATOMIC_OP_ALIAS(atomic_dec_32, _atomic_dec_32)
LEAF(_atomic_dec_32_nv)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
INT_ADDU v0, -1
@@ -80,6 +82,7 @@ LEAF(_atomic_dec_64)
li t0, -1
saad t0, (a0)
#else
+ LLSCSYNC
1: REG_LL t0, 0(a0)
nop
REG_ADDU t0, -1
@@ -93,6 +96,7 @@ END(_atomic_dec_64)
ATOMIC_OP_ALIAS(atomic_dec_64, _atomic_dec_64)
LEAF(_atomic_dec_64_nv)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
REG_ADDU v0, -1
Index: src/common/lib/libc/arch/mips/atomic/atomic_inc.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_inc.S:1.6 src/common/lib/libc/arch/mips/atomic/atomic_inc.S:1.7
--- src/common/lib/libc/arch/mips/atomic/atomic_inc.S:1.6 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_inc.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_inc.S,v 1.6 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_inc.S,v 1.7 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -29,7 +29,7 @@
#include <machine/asm.h>
#include "atomic_op_asm.h"
-RCSID("$NetBSD: atomic_inc.S,v 1.6 2020/08/01 09:26:49 skrll Exp $")
+RCSID("$NetBSD: atomic_inc.S,v 1.7 2020/08/06 10:00:21 skrll Exp $")
.text
.set noreorder
@@ -50,6 +50,7 @@ LEAF(_atomic_inc_32)
li t0, 1
saa t0, (a0)
#else
+ LLSCSYNC
1: INT_LL t0, 0(a0)
nop
INT_ADDU t0, 1
@@ -63,6 +64,7 @@ END(_atomic_inc_32)
ATOMIC_OP_ALIAS(atomic_inc_32, _atomic_inc_32)
LEAF(_atomic_inc_32_nv)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
INT_ADDU v0, 1
@@ -81,6 +83,7 @@ LEAF(_atomic_inc_64)
li t0, 1
saad t0, (a0)
#else
+ LLSCSYNC
1: REG_LL t0, 0(a0)
nop
REG_ADDU t0, 1
@@ -94,6 +97,7 @@ END(_atomic_inc_64)
ATOMIC_OP_ALIAS(atomic_inc_64, _atomic_inc_64)
LEAF(_atomic_inc_64_nv)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
REG_ADDU v0, 1
Index: src/common/lib/libc/arch/mips/atomic/atomic_swap.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_swap.S:1.6 src/common/lib/libc/arch/mips/atomic/atomic_swap.S:1.7
--- src/common/lib/libc/arch/mips/atomic/atomic_swap.S:1.6 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_swap.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_swap.S,v 1.6 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_swap.S,v 1.7 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -29,7 +29,7 @@
#include <machine/asm.h>
#include "atomic_op_asm.h"
-RCSID("$NetBSD: atomic_swap.S,v 1.6 2020/08/01 09:26:49 skrll Exp $")
+RCSID("$NetBSD: atomic_swap.S,v 1.7 2020/08/06 10:00:21 skrll Exp $")
.text
.set noreorder
@@ -44,8 +44,8 @@ RCSID("$NetBSD: atomic_swap.S,v 1.6 2020
.set nomacro
#endif /* _KERNEL_OPT */
-
LEAF(_atomic_swap_32)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
move t0, a1
@@ -60,6 +60,7 @@ ATOMIC_OP_ALIAS(atomic_swap_32, _atomic_
#if !defined(__mips_o32)
LEAF(_atomic_swap_64)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
move t0, a1
Index: src/common/lib/libc/arch/mips/atomic/atomic_and.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_and.S:1.5 src/common/lib/libc/arch/mips/atomic/atomic_and.S:1.6
--- src/common/lib/libc/arch/mips/atomic/atomic_and.S:1.5 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_and.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and.S,v 1.5 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_and.S,v 1.6 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -29,7 +29,7 @@
#include <machine/asm.h>
#include "atomic_op_asm.h"
-RCSID("$NetBSD: atomic_and.S,v 1.5 2020/08/01 09:26:49 skrll Exp $")
+RCSID("$NetBSD: atomic_and.S,v 1.6 2020/08/06 10:00:21 skrll Exp $")
.text
.set noreorder
@@ -46,6 +46,7 @@ RCSID("$NetBSD: atomic_and.S,v 1.5 2020/
LEAF(_atomic_and_32)
+ LLSCSYNC
1: INT_LL t0, 0(a0)
nop
and t0, a1
@@ -58,6 +59,7 @@ END(_atomic_and_32)
ATOMIC_OP_ALIAS(atomic_and_32, _atomic_and_32)
LEAF(_atomic_and_32_nv)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
and v0, a1
@@ -72,6 +74,7 @@ ATOMIC_OP_ALIAS(atomic_and_32_nv, _atomi
#if !defined(__mips_o32)
LEAF(_atomic_and_64)
+ LLSCSYNC
1: REG_LL t0, 0(a0)
nop
and t0, a1
@@ -84,6 +87,7 @@ END(_atomic_and_64)
ATOMIC_OP_ALIAS(atomic_and_64, _atomic_and_64)
LEAF(_atomic_and_64_nv)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
and v0, a1
Index: src/common/lib/libc/arch/mips/atomic/atomic_or.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_or.S:1.5 src/common/lib/libc/arch/mips/atomic/atomic_or.S:1.6
--- src/common/lib/libc/arch/mips/atomic/atomic_or.S:1.5 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_or.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_or.S,v 1.5 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_or.S,v 1.6 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -43,6 +43,7 @@
#endif /* _KERNEL_OPT */
LEAF(_atomic_or_32)
+ LLSCSYNC
1: INT_LL t0, 0(a0)
nop
or t0, a1
@@ -55,6 +56,7 @@ END(_atomic_or_32)
ATOMIC_OP_ALIAS(atomic_or_32, _atomic_or_32)
LEAF(_atomic_or_32_nv)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
or v0, a1
@@ -69,6 +71,7 @@ ATOMIC_OP_ALIAS(atomic_or_32_nv, _atomic
#if !defined(__mips_o32)
LEAF(_atomic_or_64)
+ LLSCSYNC
1: REG_LL t0, 0(a0)
nop
or t0, a1
@@ -81,6 +84,7 @@ END(_atomic_or_64)
ATOMIC_OP_ALIAS(atomic_or_64, _atomic_or_64)
LEAF(_atomic_or_64_nv)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
or v0, a1
Index: src/common/lib/libc/arch/mips/atomic/atomic_cas.S
diff -u src/common/lib/libc/arch/mips/atomic/atomic_cas.S:1.7 src/common/lib/libc/arch/mips/atomic/atomic_cas.S:1.8
--- src/common/lib/libc/arch/mips/atomic/atomic_cas.S:1.7 Sat Aug 1 09:26:49 2020
+++ src/common/lib/libc/arch/mips/atomic/atomic_cas.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas.S,v 1.7 2020/08/01 09:26:49 skrll Exp $ */
+/* $NetBSD: atomic_cas.S,v 1.8 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -29,7 +29,7 @@
#include <machine/asm.h>
#include "atomic_op_asm.h"
-RCSID("$NetBSD: atomic_cas.S,v 1.7 2020/08/01 09:26:49 skrll Exp $")
+RCSID("$NetBSD: atomic_cas.S,v 1.8 2020/08/06 10:00:21 skrll Exp $")
.text
.set noat
@@ -37,6 +37,7 @@ RCSID("$NetBSD: atomic_cas.S,v 1.7 2020/
.set nomacro
LEAF(_atomic_cas_32)
+ LLSCSYNC
1: INT_LL v0, 0(a0)
nop
bne v0, a1, 2f
@@ -58,6 +59,7 @@ ATOMIC_OP_ALIAS(atomic_cas_32_ni, _atomi
#if !defined(__mips_o32)
LEAF(_atomic_cas_64)
+ LLSCSYNC
1: REG_LL v0, 0(a0)
nop
bne v0, a1, 2f
Index: src/sys/arch/mips/include/asm.h
diff -u src/sys/arch/mips/include/asm.h:1.57 src/sys/arch/mips/include/asm.h:1.58
--- src/sys/arch/mips/include/asm.h:1.57 Sun Jul 26 08:08:41 2020
+++ src/sys/arch/mips/include/asm.h Thu Aug 6 10:00:20 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: asm.h,v 1.57 2020/07/26 08:08:41 simonb Exp $ */
+/* $NetBSD: asm.h,v 1.58 2020/08/06 10:00:20 skrll Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -54,6 +54,10 @@
#ifndef _MIPS_ASM_H
#define _MIPS_ASM_H
+#include "opt_cputype.h"
+#include "opt_lockdebug.h"
+#include "opt_multiprocessor.h"
+
#include <sys/cdefs.h> /* for API selection */
#include <mips/regdef.h>
@@ -513,6 +517,24 @@ _C_LABEL(x):
#define NOP_L /* nothing */
#endif
+#if defined(MULTIPROCESSOR)
+#if defined(MIPS64_OCTEON)
+ /* early cnMIPS have erratum which means 2 */
+#define LLSCSYNC sync 4; sync 4
+#define SYNC sync 4 /* sync 4 == syncw - sync all writes */
+#define BDSYNC sync 4 /* sync 4 == syncw - sync all writes */
+#else
+#define LLSCSYNC /* nothing (something?) */
+#define SYNC sync
+#define BDSYNC sync
+#endif
+#else
+#define LLSCSYNC /* nothing */
+#define SYNC /* nothing */
+#define BDSYNC nop
+#endif /* defined(MULTIPROCESSOR) */
+
+
/* CPU dependent hook for cp0 load delays */
#if defined(MIPS1) || defined(MIPS2) || defined(MIPS3)
#define MFC0_HAZARD sll $0,$0,1 /* super scalar nop */
Index: src/sys/arch/mips/mips/lock_stubs_llsc.S
diff -u src/sys/arch/mips/mips/lock_stubs_llsc.S:1.10 src/sys/arch/mips/mips/lock_stubs_llsc.S:1.11
--- src/sys/arch/mips/mips/lock_stubs_llsc.S:1.10 Sat Aug 1 07:09:41 2020
+++ src/sys/arch/mips/mips/lock_stubs_llsc.S Thu Aug 6 10:00:21 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs_llsc.S,v 1.10 2020/08/01 07:09:41 simonb Exp $ */
+/* $NetBSD: lock_stubs_llsc.S,v 1.11 2020/08/06 10:00:21 skrll Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
#include <machine/asm.h>
-RCSID("$NetBSD: lock_stubs_llsc.S,v 1.10 2020/08/01 07:09:41 simonb Exp $")
+RCSID("$NetBSD: lock_stubs_llsc.S,v 1.11 2020/08/06 10:00:21 skrll Exp $")
#include "assym.h"
@@ -45,20 +45,6 @@ RCSID("$NetBSD: lock_stubs_llsc.S,v 1.10
#define FULL
#endif
-#if defined(MULTIPROCESSOR)
-#if defined(__OCTEON__)
-#define SYNC sync 4 /* sync 4 == syncw - sync all writes */
-#define BDSYNC sync 4 /* sync 4 == syncw - sync all writes */
-#else
-#define SYNC sync
-#define BDSYNC sync
-#endif
-#else
-#define SYNC /* nothing */
-#define BDSYNC nop
-#endif /* MIPS_HAS_LLSC != 0 && defined(MULTIPROCESSOR) */
-
-
/*
* Set ISA level for the assembler.
* XXX Clean up with a macro? Same code fragment is in mipsX_subr.S too.
@@ -84,6 +70,7 @@ RCSID("$NetBSD: lock_stubs_llsc.S,v 1.10
* unsigned long old, unsigned long new);
*/
STATIC_LEAF(llsc_atomic_cas_ulong)
+ LLSCSYNC
1:
LONG_LL t0, (a0)
bne t0, a1, 2f
@@ -104,6 +91,7 @@ END(llsc_atomic_cas_ulong)
* unsigned int old, unsigned int new);
*/
STATIC_LEAF(llsc_atomic_cas_uint)
+ LLSCSYNC
1:
INT_LL t0, (a0)
bne t0, a1, 2f
@@ -133,6 +121,7 @@ STATIC_LEAF(llsc_ucas_32)
nop
move v0, zero
+ LLSCSYNC
1: ll t0, 0(a0)
bne t0, a1, 2f
move t1, a2
@@ -161,6 +150,7 @@ STATIC_LEAF(llsc_ucas_64)
nop
move v0, zero
+ LLSCSYNC
1: lld t0, 0(a0)
bne t0, a1, 2f
move t1, a2
@@ -187,6 +177,7 @@ END(llsc_ucaserr)
* void mutex_enter(kmutex_t *mtx);
*/
STATIC_LEAF(llsc_mutex_enter)
+ LLSCSYNC
PTR_LL t0, MTX_OWNER(a0)
1:
bnez t0, 2f
@@ -205,6 +196,7 @@ END(llsc_mutex_enter)
* void mutex_exit(kmutex_t *mtx);
*/
STATIC_LEAF(llsc_mutex_exit)
+ LLSCSYNC
PTR_LL t0, MTX_OWNER(a0)
SYNC
1:
@@ -268,6 +260,7 @@ STATIC_NESTED(llsc_mutex_spin_enter, CAL
nop
#endif /* PARANOIA */
#if defined(FULL)
+ LLSCSYNC
INT_LL t3, MTX_LOCK(t0)
3:
bnez t3, 4f
