Module Name: src Committed By: riastradh Date: Wed Jun 29 22:27:01 UTC 2022
Modified Files: src/sys/kern: kern_condvar.c kern_sleepq.c kern_synch.c kern_timeout.c kern_turnstile.c sys_lwp.c sys_select.c src/sys/sys: sleepq.h Log Message: sleepq(9): Pass syncobj through to sleepq_block. Previously the usage pattern was: sleepq_enter(sq, l, lock); // locks l ... sleepq_enqueue(sq, ..., sobj, ...); // assumes l locked, sets l_syncobj ... (*) sleepq_block(...); // unlocks l As long as l remains locked from sleepq_enter to sleepq_block, l_syncobj is stable, and sleepq_block uses it via ktrcsw to determine whether the sleep is on a mutex in order to avoid creating ktrace context-switch records (which involves allocation which is forbidden in softint context, while taking and even sleeping for a mutex is allowed). However, in turnstile_block, the logic at (*) also involves turnstile_lendpri, which sometimes unlocks and relocks l. At that point, another thread can swoop in and sleepq_remove l, which sets l_syncobj to sched_syncobj. If that happens, ktrcsw does what is forbidden -- tries to allocate a ktrace record for the context switch. As an optimization, sleepq_block or turnstile_block could stop early if it detects that l_syncobj doesn't match -- we've already been requested to wake up at this point so there's no need to mi_switch. (And then it would be unnecessary to pass the syncobj through sleepq_block, because l_syncobj would remain stable.) But I'll leave that to another change. Reported-by: syzbot+8b9d7b066c32dbcdc...@syzkaller.appspotmail.com To generate a diff of this commit: cvs rdiff -u -r1.53 -r1.54 src/sys/kern/kern_condvar.c cvs rdiff -u -r1.72 -r1.73 src/sys/kern/kern_sleepq.c cvs rdiff -u -r1.350 -r1.351 src/sys/kern/kern_synch.c cvs rdiff -u -r1.69 -r1.70 src/sys/kern/kern_timeout.c cvs rdiff -u -r1.43 -r1.44 src/sys/kern/kern_turnstile.c cvs rdiff -u -r1.82 -r1.83 src/sys/kern/sys_lwp.c cvs rdiff -u -r1.59 -r1.60 src/sys/kern/sys_select.c cvs rdiff -u -r1.34 -r1.35 src/sys/sys/sleepq.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/kern/kern_condvar.c diff -u src/sys/kern/kern_condvar.c:1.53 src/sys/kern/kern_condvar.c:1.54 --- src/sys/kern/kern_condvar.c:1.53 Sun Nov 1 20:55:15 2020 +++ src/sys/kern/kern_condvar.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_condvar.c,v 1.53 2020/11/01 20:55:15 christos Exp $ */ +/* $NetBSD: kern_condvar.c,v 1.54 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2006, 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc. @@ -34,7 +34,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_condvar.c,v 1.53 2020/11/01 20:55:15 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_condvar.c,v 1.54 2022/06/29 22:27:01 riastradh Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -171,7 +171,7 @@ cv_wait(kcondvar_t *cv, kmutex_t *mtx) KASSERT(mutex_owned(mtx)); cv_enter(cv, mtx, l, false); - (void)sleepq_block(0, false); + (void)sleepq_block(0, false, &cv_syncobj); mutex_enter(mtx); } @@ -192,7 +192,7 @@ cv_wait_sig(kcondvar_t *cv, kmutex_t *mt KASSERT(mutex_owned(mtx)); cv_enter(cv, mtx, l, true); - error = sleepq_block(0, true); + error = sleepq_block(0, true, &cv_syncobj); mutex_enter(mtx); return error; } @@ -215,7 +215,7 @@ cv_timedwait(kcondvar_t *cv, kmutex_t *m KASSERT(mutex_owned(mtx)); cv_enter(cv, mtx, l, false); - error = sleepq_block(timo, false); + error = sleepq_block(timo, false, &cv_syncobj); mutex_enter(mtx); return error; } @@ -240,7 +240,7 @@ cv_timedwait_sig(kcondvar_t *cv, kmutex_ KASSERT(mutex_owned(mtx)); cv_enter(cv, mtx, l, true); - error = sleepq_block(timo, true); + error = sleepq_block(timo, true, &cv_syncobj); mutex_enter(mtx); return error; } Index: src/sys/kern/kern_sleepq.c diff -u src/sys/kern/kern_sleepq.c:1.72 src/sys/kern/kern_sleepq.c:1.73 --- src/sys/kern/kern_sleepq.c:1.72 Wed Jun 29 22:10:43 2022 +++ src/sys/kern/kern_sleepq.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_sleepq.c,v 1.72 2022/06/29 22:10:43 riastradh Exp $ */ +/* $NetBSD: kern_sleepq.c,v 1.73 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2006, 2007, 2008, 2009, 2019, 2020 The NetBSD Foundation, Inc. @@ -35,7 +35,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.72 2022/06/29 22:10:43 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.73 2022/06/29 22:27:01 riastradh Exp $"); #include <sys/param.h> #include <sys/kernel.h> @@ -302,14 +302,13 @@ sleepq_uncatch(lwp_t *l) * timo is a timeout in ticks. timo = 0 specifies an infinite timeout. */ int -sleepq_block(int timo, bool catch_p) +sleepq_block(int timo, bool catch_p, struct syncobj *syncobj) { int error = 0, sig; struct proc *p; lwp_t *l = curlwp; bool early = false; int biglocks = l->l_biglocks; - struct syncobj *syncobj = l->l_syncobj; ktrcsw(1, 0, syncobj); Index: src/sys/kern/kern_synch.c diff -u src/sys/kern/kern_synch.c:1.350 src/sys/kern/kern_synch.c:1.351 --- src/sys/kern/kern_synch.c:1.350 Thu Mar 10 12:21:25 2022 +++ src/sys/kern/kern_synch.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_synch.c,v 1.350 2022/03/10 12:21:25 riastradh Exp $ */ +/* $NetBSD: kern_synch.c,v 1.351 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019, 2020 @@ -69,7 +69,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.350 2022/03/10 12:21:25 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.351 2022/06/29 22:27:01 riastradh Exp $"); #include "opt_kstack.h" #include "opt_dtrace.h" @@ -188,7 +188,7 @@ tsleep(wchan_t ident, pri_t priority, co sq = sleeptab_lookup(&sleeptab, ident, &mp); sleepq_enter(sq, l, mp); sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj, catch_p); - return sleepq_block(timo, catch_p); + return sleepq_block(timo, catch_p, &sleep_syncobj); } int @@ -215,7 +215,7 @@ mtsleep(wchan_t ident, pri_t priority, c sleepq_enter(sq, l, mp); sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj, catch_p); mutex_exit(mtx); - error = sleepq_block(timo, catch_p); + error = sleepq_block(timo, catch_p, &sleep_syncobj); if ((priority & PNORELOCK) == 0) mutex_enter(mtx); @@ -243,7 +243,7 @@ kpause(const char *wmesg, bool intr, int lwp_lock(l); KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); sleepq_enqueue(NULL, l, wmesg, &kpause_syncobj, intr); - error = sleepq_block(timo, intr); + error = sleepq_block(timo, intr, &kpause_syncobj); if (mtx != NULL) mutex_enter(mtx); Index: src/sys/kern/kern_timeout.c diff -u src/sys/kern/kern_timeout.c:1.69 src/sys/kern/kern_timeout.c:1.70 --- src/sys/kern/kern_timeout.c:1.69 Wed Mar 30 17:02:02 2022 +++ src/sys/kern/kern_timeout.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_timeout.c,v 1.69 2022/03/30 17:02:02 riastradh Exp $ */ +/* $NetBSD: kern_timeout.c,v 1.70 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2003, 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc. @@ -59,7 +59,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.69 2022/03/30 17:02:02 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.70 2022/06/29 22:27:01 riastradh Exp $"); /* * Timeouts are kept in a hierarchical timing wheel. The c_time is the @@ -543,7 +543,7 @@ callout_wait(callout_impl_t *c, void *in sleepq_enter(&cc->cc_sleepq, l, cc->cc_lock); sleepq_enqueue(&cc->cc_sleepq, cc, "callout", &sleep_syncobj, false); - sleepq_block(0, false); + sleepq_block(0, false, &sleep_syncobj); } /* Index: src/sys/kern/kern_turnstile.c diff -u src/sys/kern/kern_turnstile.c:1.43 src/sys/kern/kern_turnstile.c:1.44 --- src/sys/kern/kern_turnstile.c:1.43 Sat May 28 22:08:46 2022 +++ src/sys/kern/kern_turnstile.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_turnstile.c,v 1.43 2022/05/28 22:08:46 andvar Exp $ */ +/* $NetBSD: kern_turnstile.c,v 1.44 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2002, 2006, 2007, 2009, 2019, 2020 @@ -61,7 +61,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.43 2022/05/28 22:08:46 andvar Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.44 2022/06/29 22:27:01 riastradh Exp $"); #include <sys/param.h> #include <sys/lockdebug.h> @@ -435,7 +435,7 @@ turnstile_block(turnstile_t *ts, int q, KPREEMPT_DISABLE(l); KASSERT(lock == l->l_mutex); turnstile_lendpri(l); - sleepq_block(0, false); + sleepq_block(0, false, sobj); l->l_kpribase = obase; KPREEMPT_ENABLE(l); } Index: src/sys/kern/sys_lwp.c diff -u src/sys/kern/sys_lwp.c:1.82 src/sys/kern/sys_lwp.c:1.83 --- src/sys/kern/sys_lwp.c:1.82 Sat May 23 23:42:43 2020 +++ src/sys/kern/sys_lwp.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: sys_lwp.c,v 1.82 2020/05/23 23:42:43 ad Exp $ */ +/* $NetBSD: sys_lwp.c,v 1.83 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2001, 2006, 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc. @@ -35,7 +35,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.82 2020/05/23 23:42:43 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.83 2022/06/29 22:27:01 riastradh Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -533,7 +533,7 @@ lwp_park(clockid_t clock_id, int flags, } l->l_biglocks = 0; sleepq_enqueue(NULL, l, "parked", &lwp_park_syncobj, true); - error = sleepq_block(timo, true); + error = sleepq_block(timo, true, &lwp_park_syncobj); switch (error) { case EWOULDBLOCK: error = ETIMEDOUT; Index: src/sys/kern/sys_select.c diff -u src/sys/kern/sys_select.c:1.59 src/sys/kern/sys_select.c:1.60 --- src/sys/kern/sys_select.c:1.59 Sat Apr 9 23:52:05 2022 +++ src/sys/kern/sys_select.c Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: sys_select.c,v 1.59 2022/04/09 23:52:05 riastradh Exp $ */ +/* $NetBSD: sys_select.c,v 1.60 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2007, 2008, 2009, 2010, 2019, 2020 The NetBSD Foundation, Inc. @@ -84,7 +84,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.59 2022/04/09 23:52:05 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.60 2022/06/29 22:27:01 riastradh Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -322,7 +322,7 @@ state_check: l->l_kpriority = true; sleepq_enter(&sc->sc_sleepq, l, lock); sleepq_enqueue(&sc->sc_sleepq, sc, opname, &select_sobj, true); - error = sleepq_block(timo, true); + error = sleepq_block(timo, true, &select_sobj); if (error != 0) { break; } Index: src/sys/sys/sleepq.h diff -u src/sys/sys/sleepq.h:1.34 src/sys/sys/sleepq.h:1.35 --- src/sys/sys/sleepq.h:1.34 Sun Nov 1 20:56:13 2020 +++ src/sys/sys/sleepq.h Wed Jun 29 22:27:01 2022 @@ -1,4 +1,4 @@ -/* $NetBSD: sleepq.h,v 1.34 2020/11/01 20:56:13 christos Exp $ */ +/* $NetBSD: sleepq.h,v 1.35 2022/06/29 22:27:01 riastradh Exp $ */ /*- * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2019, 2020 @@ -60,7 +60,7 @@ void sleepq_wake(sleepq_t *, wchan_t, u_ int sleepq_abort(kmutex_t *, int); void sleepq_changepri(lwp_t *, pri_t); void sleepq_lendpri(lwp_t *, pri_t); -int sleepq_block(int, bool); +int sleepq_block(int, bool, struct syncobj *); #ifdef _KERNEL typedef union {