Module Name: src Committed By: riastradh Date: Tue Dec 3 15:21:00 UTC 2019
Modified Files: src/sys/arch/arm/arm: arm_machdep.c src/sys/arch/mips/mips: cpu_subr.c src/sys/arch/sparc/sparc: intr.c src/sys/arch/sparc64/sparc64: machdep.c src/sys/arch/usermode/dev: cpu.c src/sys/arch/x86/x86: pmap.c x86_machdep.c src/sys/kern: kern_lock.c subr_pool.c Log Message: Use __insn_barrier to enforce ordering in l_ncsw loops. (Only need ordering observable by interruption, not by other CPUs.) To generate a diff of this commit: cvs rdiff -u -r1.57 -r1.58 src/sys/arch/arm/arm/arm_machdep.c cvs rdiff -u -r1.39 -r1.40 src/sys/arch/mips/mips/cpu_subr.c cvs rdiff -u -r1.122 -r1.123 src/sys/arch/sparc/sparc/intr.c cvs rdiff -u -r1.294 -r1.295 src/sys/arch/sparc64/sparc64/machdep.c cvs rdiff -u -r1.82 -r1.83 src/sys/arch/usermode/dev/cpu.c cvs rdiff -u -r1.341 -r1.342 src/sys/arch/x86/x86/pmap.c cvs rdiff -u -r1.132 -r1.133 src/sys/arch/x86/x86/x86_machdep.c cvs rdiff -u -r1.163 -r1.164 src/sys/kern/kern_lock.c cvs rdiff -u -r1.262 -r1.263 src/sys/kern/subr_pool.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/arm/arm/arm_machdep.c diff -u src/sys/arch/arm/arm/arm_machdep.c:1.57 src/sys/arch/arm/arm/arm_machdep.c:1.58 --- src/sys/arch/arm/arm/arm_machdep.c:1.57 Sun Dec 1 14:52:13 2019 +++ src/sys/arch/arm/arm/arm_machdep.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: arm_machdep.c,v 1.57 2019/12/01 14:52:13 ad Exp $ */ +/* $NetBSD: arm_machdep.c,v 1.58 2019/12/03 15:20:59 riastradh Exp $ */ /* * Copyright (c) 2001 Wasabi Systems, Inc. @@ -80,7 +80,7 @@ #include <sys/param.h> -__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.57 2019/12/01 14:52:13 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.58 2019/12/03 15:20:59 riastradh Exp $"); #include <sys/exec.h> #include <sys/proc.h> @@ -270,10 +270,12 @@ cpu_intr_p(void) l = curlwp; do { ncsw = l->l_ncsw; + __insn_barrier(); idepth = l->l_cpu->ci_intr_depth; #ifdef __HAVE_PIC_FAST_SOFTINTS cpl = ci->ci_cpl; #endif + __insn_barrier(); } while (__predict_false(ncsw != l->l_ncsw)); #ifdef __HAVE_PIC_FAST_SOFTINTS Index: src/sys/arch/mips/mips/cpu_subr.c diff -u src/sys/arch/mips/mips/cpu_subr.c:1.39 src/sys/arch/mips/mips/cpu_subr.c:1.40 --- src/sys/arch/mips/mips/cpu_subr.c:1.39 Mon Dec 2 23:22:43 2019 +++ src/sys/arch/mips/mips/cpu_subr.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $ */ +/* $NetBSD: cpu_subr.c,v 1.40 2019/12/03 15:20:59 riastradh Exp $ */ /*- * Copyright (c) 2010, 2019 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.39 2019/12/02 23:22:43 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.40 2019/12/03 15:20:59 riastradh Exp $"); #include "opt_cputype.h" #include "opt_ddb.h" @@ -610,7 +610,9 @@ cpu_intr_p(void) l = curlwp; do { ncsw = l->l_ncsw; + __insn_barrier(); idepth = l->l_cpu->ci_idepth; + __insn_barrier(); } while (__predict_false(ncsw != l->l_ncsw)); return idepth != 0; Index: src/sys/arch/sparc/sparc/intr.c diff -u src/sys/arch/sparc/sparc/intr.c:1.122 src/sys/arch/sparc/sparc/intr.c:1.123 --- src/sys/arch/sparc/sparc/intr.c:1.122 Sun Dec 1 14:52:14 2019 +++ src/sys/arch/sparc/sparc/intr.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: intr.c,v 1.122 2019/12/01 14:52:14 ad Exp $ */ +/* $NetBSD: intr.c,v 1.123 2019/12/03 15:20:59 riastradh Exp $ */ /* * Copyright (c) 1992, 1993 @@ -41,7 +41,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.122 2019/12/01 14:52:14 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.123 2019/12/03 15:20:59 riastradh Exp $"); #include "opt_multiprocessor.h" #include "opt_sparc_arch.h" @@ -896,7 +896,9 @@ cpu_intr_p(void) l = curlwp; do { ncsw = l->l_ncsw; + __insn_barrier(); idepth = l->l_cpu->ci_idepth; + __insn_barrier(); } while (__predict_false(ncsw != l->l_ncsw)); return idepth != 0; Index: src/sys/arch/sparc64/sparc64/machdep.c diff -u src/sys/arch/sparc64/sparc64/machdep.c:1.294 src/sys/arch/sparc64/sparc64/machdep.c:1.295 --- src/sys/arch/sparc64/sparc64/machdep.c:1.294 Sun Dec 1 14:52:14 2019 +++ src/sys/arch/sparc64/sparc64/machdep.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: machdep.c,v 1.294 2019/12/01 14:52:14 ad Exp $ */ +/* $NetBSD: machdep.c,v 1.295 2019/12/03 15:20:59 riastradh Exp $ */ /*- * Copyright (c) 1996, 1997, 1998, 2019 The NetBSD Foundation, Inc. @@ -71,7 +71,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.294 2019/12/01 14:52:14 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.295 2019/12/03 15:20:59 riastradh Exp $"); #include "opt_ddb.h" #include "opt_multiprocessor.h" @@ -2661,7 +2661,9 @@ cpu_intr_p(void) l = curlwp; do { ncsw = l->l_ncsw; + __insn_barrier(); idepth = l->l_cpu->ci_idepth; + __insn_barrier(); } while (__predict_false(ncsw != l->l_ncsw)); return idepth >= 0; Index: src/sys/arch/usermode/dev/cpu.c diff -u src/sys/arch/usermode/dev/cpu.c:1.82 src/sys/arch/usermode/dev/cpu.c:1.83 --- src/sys/arch/usermode/dev/cpu.c:1.82 Sun Dec 1 14:52:14 2019 +++ src/sys/arch/usermode/dev/cpu.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.c,v 1.82 2019/12/01 14:52:14 ad Exp $ */ +/* $NetBSD: cpu.c,v 1.83 2019/12/03 15:20:59 riastradh Exp $ */ /*- * Copyright (c) 2007 Jared D. McNeill <jmcne...@invisible.ca> @@ -30,7 +30,7 @@ #include "opt_hz.h" #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.82 2019/12/01 14:52:14 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.83 2019/12/03 15:20:59 riastradh Exp $"); #include <sys/param.h> #include <sys/conf.h> @@ -535,7 +535,9 @@ cpu_intr_p(void) l = curlwp; do { ncsw = l->l_ncsw; + __insn_barrier(); idepth = l->l_cpu->ci_idepth; + __insn_barrier(); } while (__predict_false(ncsw != l->l_ncsw)); return idepth >= 0; Index: src/sys/arch/x86/x86/pmap.c diff -u src/sys/arch/x86/x86/pmap.c:1.341 src/sys/arch/x86/x86/pmap.c:1.342 --- src/sys/arch/x86/x86/pmap.c:1.341 Sat Nov 16 10:19:29 2019 +++ src/sys/arch/x86/x86/pmap.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.c,v 1.341 2019/11/16 10:19:29 maxv Exp $ */ +/* $NetBSD: pmap.c,v 1.342 2019/12/03 15:20:59 riastradh Exp $ */ /* * Copyright (c) 2008, 2010, 2016, 2017 The NetBSD Foundation, Inc. @@ -130,7 +130,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.341 2019/11/16 10:19:29 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.342 2019/12/03 15:20:59 riastradh Exp $"); #include "opt_user_ldt.h" #include "opt_lockdebug.h" @@ -2893,6 +2893,7 @@ pmap_load(void) } l = ci->ci_curlwp; ncsw = l->l_ncsw; + __insn_barrier(); /* should be able to take ipis. */ KASSERT(ci->ci_ilevel < IPL_HIGH); @@ -2974,6 +2975,7 @@ pmap_load(void) */ pmap_destroy(oldpmap); + __insn_barrier(); if (l->l_ncsw != ncsw) { goto retry; } Index: src/sys/arch/x86/x86/x86_machdep.c diff -u src/sys/arch/x86/x86/x86_machdep.c:1.132 src/sys/arch/x86/x86/x86_machdep.c:1.133 --- src/sys/arch/x86/x86/x86_machdep.c:1.132 Tue Dec 3 11:50:45 2019 +++ src/sys/arch/x86/x86/x86_machdep.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: x86_machdep.c,v 1.132 2019/12/03 11:50:45 hannken Exp $ */ +/* $NetBSD: x86_machdep.c,v 1.133 2019/12/03 15:20:59 riastradh Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, @@ -31,7 +31,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.132 2019/12/03 11:50:45 hannken Exp $"); +__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.133 2019/12/03 15:20:59 riastradh Exp $"); #include "opt_modular.h" #include "opt_physmem.h" @@ -350,7 +350,7 @@ bool cpu_intr_p(void) { uint64_t ncsw; - volatile int idepth; + int idepth; lwp_t *l; l = curlwp; @@ -360,7 +360,9 @@ cpu_intr_p(void) } do { ncsw = l->l_ncsw; + __insn_barrier(); idepth = l->l_cpu->ci_idepth; + __insn_barrier(); } while (__predict_false(ncsw != l->l_ncsw)); return idepth >= 0; Index: src/sys/kern/kern_lock.c diff -u src/sys/kern/kern_lock.c:1.163 src/sys/kern/kern_lock.c:1.164 --- src/sys/kern/kern_lock.c:1.163 Thu May 9 05:00:31 2019 +++ src/sys/kern/kern_lock.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_lock.c,v 1.163 2019/05/09 05:00:31 ozaki-r Exp $ */ +/* $NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $ */ /*- * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. @@ -31,7 +31,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.163 2019/05/09 05:00:31 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $"); #include <sys/param.h> #include <sys/proc.h> @@ -75,7 +75,9 @@ assert_sleepable(void) */ do { pctr = lwp_pctr(); + __insn_barrier(); idle = CURCPU_IDLE_P(); + __insn_barrier(); } while (pctr != lwp_pctr()); reason = NULL; Index: src/sys/kern/subr_pool.c diff -u src/sys/kern/subr_pool.c:1.262 src/sys/kern/subr_pool.c:1.263 --- src/sys/kern/subr_pool.c:1.262 Thu Nov 14 16:23:53 2019 +++ src/sys/kern/subr_pool.c Tue Dec 3 15:20:59 2019 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.262 2019/11/14 16:23:53 maxv Exp $ */ +/* $NetBSD: subr_pool.c,v 1.263 2019/12/03 15:20:59 riastradh Exp $ */ /* * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018 @@ -33,7 +33,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.262 2019/11/14 16:23:53 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.263 2019/12/03 15:20:59 riastradh Exp $"); #ifdef _KERNEL_OPT #include "opt_ddb.h" @@ -2460,6 +2460,7 @@ pool_cache_get_slow(pool_cache_cpu_t *cc */ if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { ncsw = curlwp->l_ncsw; + __insn_barrier(); mutex_enter(&pc->pc_lock); pc->pc_contended++; @@ -2468,6 +2469,7 @@ pool_cache_get_slow(pool_cache_cpu_t *cc * our view of the per-CPU data is invalid: * retry. */ + __insn_barrier(); if (curlwp->l_ncsw != ncsw) { mutex_exit(&pc->pc_lock); return true; @@ -2625,6 +2627,7 @@ pool_cache_put_slow(pool_cache_cpu_t *cc pcg = NULL; cc->cc_misses++; ncsw = l->l_ncsw; + __insn_barrier(); /* * If there are no empty groups in the cache then allocate one @@ -2638,6 +2641,7 @@ pool_cache_put_slow(pool_cache_cpu_t *cc * If pool_get() blocked, then our view of * the per-CPU data is invalid: retry. */ + __insn_barrier(); if (__predict_false(l->l_ncsw != ncsw)) { if (pcg != NULL) { pool_put(pc->pc_pcgpool, pcg); @@ -2659,6 +2663,7 @@ pool_cache_put_slow(pool_cache_cpu_t *cc * If we context switched while locking, then our view of * the per-CPU data is invalid: retry. */ + __insn_barrier(); if (__predict_false(l->l_ncsw != ncsw)) { mutex_exit(&pc->pc_lock); if (pcg != NULL) {