Module Name: src Committed By: riastradh Date: Thu Feb 23 14:56:00 UTC 2023
Modified Files: src/sys/arch/evbmips/ingenic: cpu_startup.S src/sys/arch/mips/include: asm.h src/sys/arch/mips/mips: locore.S locore_mips3.S Log Message: mips: Add missing barriers in cpu_switchto. Details in comments. PR kern/57240 XXX pullup-8 XXX pullup-9 XXX pullup-10 To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 src/sys/arch/evbmips/ingenic/cpu_startup.S cvs rdiff -u -r1.73 -r1.74 src/sys/arch/mips/include/asm.h cvs rdiff -u -r1.228 -r1.229 src/sys/arch/mips/mips/locore.S cvs rdiff -u -r1.115 -r1.116 src/sys/arch/mips/mips/locore_mips3.S Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/evbmips/ingenic/cpu_startup.S diff -u src/sys/arch/evbmips/ingenic/cpu_startup.S:1.1 src/sys/arch/evbmips/ingenic/cpu_startup.S:1.2 --- src/sys/arch/evbmips/ingenic/cpu_startup.S:1.1 Fri Jan 29 01:54:14 2016 +++ src/sys/arch/evbmips/ingenic/cpu_startup.S Thu Feb 23 14:56:00 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_startup.S,v 1.1 2016/01/29 01:54:14 macallan Exp $ */ +/* $NetBSD: cpu_startup.S,v 1.2 2023/02/23 14:56:00 riastradh Exp $ */ /*- * Copyright (c) 2015 Michael Lorenz @@ -33,7 +33,7 @@ #include <sys/endian.h> #include <mips/asm.h> -RCSID("$NetBSD: cpu_startup.S,v 1.1 2016/01/29 01:54:14 macallan Exp $"); +RCSID("$NetBSD: cpu_startup.S,v 1.2 2023/02/23 14:56:00 riastradh Exp $"); #ifdef MULTIPROCESSOR @@ -56,6 +56,11 @@ NESTED_NOPROFILE(ingenic_trampoline, 0, nop beqz MIPS_CURLWP, 1b nop + /* + * No membar needed because we're not switching from a + * previous lwp, and the idle lwp we're switching to can't be + * holding locks already; see cpu_switchto. + */ PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(a0) li v0, 0 Index: src/sys/arch/mips/include/asm.h diff -u src/sys/arch/mips/include/asm.h:1.73 src/sys/arch/mips/include/asm.h:1.74 --- src/sys/arch/mips/include/asm.h:1.73 Mon Feb 20 13:30:47 2023 +++ src/sys/arch/mips/include/asm.h Thu Feb 23 14:56:00 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: asm.h,v 1.73 2023/02/20 13:30:47 riastradh Exp $ */ +/* $NetBSD: asm.h,v 1.74 2023/02/23 14:56:00 riastradh Exp $ */ /* * Copyright (c) 1992, 1993 @@ -633,6 +633,28 @@ _C_LABEL(x): #define SYNC_PLUNGER /* nothing */ #endif +/* + * Store-before-load barrier. Do not use this unless you know what + * you're doing. + */ +#ifdef MULTIPROCESSOR +#define SYNC_DEKKER sync +#else +#define SYNC_DEKKER /* nothing */ +#endif + +/* + * Store-before-store and load-before-load barriers. These could be + * made weaker than release (load/store-before-store) and acquire + * (load-before-load/store) barriers, and newer MIPS does have + * instruction encodings for finer-grained barriers like this, but I + * dunno how to appropriately conditionalize their use or get the + * assembler to be happy with them, so we'll use these definitions for + * now. + */ +#define SYNC_PRODUCER SYNC_REL +#define SYNC_CONSUMER SYNC_ACQ + /* CPU dependent hook for cp0 load delays */ #if defined(MIPS1) || defined(MIPS2) || defined(MIPS3) #define MFC0_HAZARD sll $0,$0,1 /* super scalar nop */ Index: src/sys/arch/mips/mips/locore.S diff -u src/sys/arch/mips/mips/locore.S:1.228 src/sys/arch/mips/mips/locore.S:1.229 --- src/sys/arch/mips/mips/locore.S:1.228 Sat May 29 12:35:27 2021 +++ src/sys/arch/mips/mips/locore.S Thu Feb 23 14:56:00 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: locore.S,v 1.228 2021/05/29 12:35:27 simonb Exp $ */ +/* $NetBSD: locore.S,v 1.229 2023/02/23 14:56:00 riastradh Exp $ */ /* * Copyright (c) 1992, 1993 @@ -63,7 +63,7 @@ #include <mips/trap.h> #include <mips/locore.h> -RCSID("$NetBSD: locore.S,v 1.228 2021/05/29 12:35:27 simonb Exp $") +RCSID("$NetBSD: locore.S,v 1.229 2023/02/23 14:56:00 riastradh Exp $") #include "assym.h" @@ -286,7 +286,28 @@ NESTED(cpu_switchto, CALLFRAME_SIZ, ra) PTR_L t2, L_CPU(MIPS_CURLWP) nop # patchable load delay slot + + /* + * Issue barriers to coordinate mutex_exit on this CPU with + * mutex_vector_enter on another CPU. + * + * 1. Any prior mutex_exit by oldlwp must be visible to other + * CPUs before we set ci_curlwp := newlwp on this one, + * requiring a store-before-store barrier. + * + * 2. ci_curlwp := newlwp must be visible on all other CPUs + * before any subsequent mutex_exit by newlwp can even test + * whether there might be waiters, requiring a + * store-before-load barrier. + * + * See kern_mutex.c for details -- this is necessary for + * adaptive mutexes to detect whether the lwp is on the CPU in + * order to safely block without requiring atomic r/m/w in + * mutex_exit. + */ + SYNC_PRODUCER /* XXX fixup to nop for uniprocessor boot */ PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(t2) + SYNC_DEKKER /* XXX fixup to nop for uniprocessor boot */ /* Check for restartable atomic sequences (RAS) */ PTR_L a0, L_PROC(MIPS_CURLWP) # argument to ras_lookup @@ -406,7 +427,9 @@ NESTED(softint_fast_dispatch, CALLFRAME_ move MIPS_CURLWP, a0 # switch to softint lwp PTR_L s1, L_CPU(MIPS_CURLWP) # get curcpu() nop # patchable load delay slot + SYNC_PRODUCER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */ PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(s1) # ... + SYNC_DEKKER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */ move s2, sp # remember sp move s3, t0 # remember curpcb @@ -417,7 +440,9 @@ NESTED(softint_fast_dispatch, CALLFRAME_ move sp, s2 # restore stack move MIPS_CURLWP, s0 # restore curlwp + SYNC_PRODUCER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */ PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(s1) # .... + SYNC_DEKKER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */ REG_L ra, CALLFRAME_RA(sp) # load early since we use it Index: src/sys/arch/mips/mips/locore_mips3.S diff -u src/sys/arch/mips/mips/locore_mips3.S:1.115 src/sys/arch/mips/mips/locore_mips3.S:1.116 --- src/sys/arch/mips/mips/locore_mips3.S:1.115 Sun May 24 07:15:24 2020 +++ src/sys/arch/mips/mips/locore_mips3.S Thu Feb 23 14:56:00 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: locore_mips3.S,v 1.115 2020/05/24 07:15:24 simonb Exp $ */ +/* $NetBSD: locore_mips3.S,v 1.116 2023/02/23 14:56:00 riastradh Exp $ */ /* * Copyright (c) 1997 Jonathan Stone (hereinafter referred to as the author) @@ -92,7 +92,7 @@ #include <mips/asm.h> #include <mips/cpuregs.h> -RCSID("$NetBSD: locore_mips3.S,v 1.115 2020/05/24 07:15:24 simonb Exp $") +RCSID("$NetBSD: locore_mips3.S,v 1.116 2023/02/23 14:56:00 riastradh Exp $") #include "assym.h" @@ -809,6 +809,11 @@ NESTED_NOPROFILE(cpu_trampoline, 0, ra) nop beqz MIPS_CURLWP, 1b nop + /* + * No membar needed because we're not switching from a + * previous lwp, and the idle lwp we're switching to can't be + * holding locks already; see cpu_switchto. + */ PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(a0) #ifdef _LP64