Module Name: src
Committed By: ad
Date: Fri Dec 6 21:36:11 UTC 2019
Modified Files:
src/sys/kern: kern_exec.c kern_exit.c kern_idle.c kern_lwp.c kern_sig.c
kern_sleepq.c kern_softint.c kern_synch.c
src/sys/sys: sched.h
Log Message:
Make it possible to call mi_switch() and immediately switch to another CPU.
This seems to take about 3us on my Intel system. Two changes required:
- Have the caller to mi_switch() be responsible for calling spc_lock().
- Avoid using l->l_cpu in mi_switch().
While here:
- Add a couple of calls to membar_enter()
- Have the idle LWP set itself to LSIDL, to match softint_thread().
- Remove unused return value from mi_switch().
To generate a diff of this commit:
cvs rdiff -u -r1.484 -r1.485 src/sys/kern/kern_exec.c
cvs rdiff -u -r1.277 -r1.278 src/sys/kern/kern_exit.c
cvs rdiff -u -r1.27 -r1.28 src/sys/kern/kern_idle.c
cvs rdiff -u -r1.216 -r1.217 src/sys/kern/kern_lwp.c
cvs rdiff -u -r1.380 -r1.381 src/sys/kern/kern_sig.c
cvs rdiff -u -r1.53 -r1.54 src/sys/kern/kern_sleepq.c
cvs rdiff -u -r1.54 -r1.55 src/sys/kern/kern_softint.c
cvs rdiff -u -r1.328 -r1.329 src/sys/kern/kern_synch.c
cvs rdiff -u -r1.79 -r1.80 src/sys/sys/sched.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/kern_exec.c
diff -u src/sys/kern/kern_exec.c:1.484 src/sys/kern/kern_exec.c:1.485
--- src/sys/kern/kern_exec.c:1.484 Sat Nov 23 19:42:52 2019
+++ src/sys/kern/kern_exec.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_exec.c,v 1.484 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: kern_exec.c,v 1.485 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_exec.c,v 1.484 2019/11/23 19:42:52 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_exec.c,v 1.485 2019/12/06 21:36:10 ad Exp $");
#include "opt_exec.h"
#include "opt_execfmt.h"
@@ -1363,6 +1363,7 @@ execve_runproc(struct lwp *l, struct exe
mutex_exit(p->p_lock);
mutex_exit(proc_lock);
lwp_lock(l);
+ spc_lock(l->l_cpu);
mi_switch(l);
ksiginfo_queue_drain(&kq);
KERNEL_LOCK(l->l_biglocks, l);
Index: src/sys/kern/kern_exit.c
diff -u src/sys/kern/kern_exit.c:1.277 src/sys/kern/kern_exit.c:1.278
--- src/sys/kern/kern_exit.c:1.277 Thu Oct 3 22:48:44 2019
+++ src/sys/kern/kern_exit.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_exit.c,v 1.277 2019/10/03 22:48:44 kamil Exp $ */
+/* $NetBSD: kern_exit.c,v 1.278 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 1998, 1999, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_exit.c,v 1.277 2019/10/03 22:48:44 kamil Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_exit.c,v 1.278 2019/12/06 21:36:10 ad Exp $");
#include "opt_ktrace.h"
#include "opt_dtrace.h"
@@ -245,6 +245,7 @@ exit1(struct lwp *l, int exitcode, int s
lwp_unlock(l);
mutex_exit(p->p_lock);
lwp_lock(l);
+ spc_lock(l->l_cpu);
mi_switch(l);
KERNEL_LOCK(l->l_biglocks, l);
mutex_enter(p->p_lock);
Index: src/sys/kern/kern_idle.c
diff -u src/sys/kern/kern_idle.c:1.27 src/sys/kern/kern_idle.c:1.28
--- src/sys/kern/kern_idle.c:1.27 Sun Dec 1 15:34:46 2019
+++ src/sys/kern/kern_idle.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_idle.c,v 1.27 2019/12/01 15:34:46 ad Exp $ */
+/* $NetBSD: kern_idle.c,v 1.28 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c)2002, 2006, 2007 YAMAMOTO Takashi,
@@ -28,7 +28,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.27 2019/12/01 15:34:46 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.28 2019/12/06 21:36:10 ad Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -93,6 +93,8 @@ idle_loop(void *dummy)
}
KASSERT(l->l_mutex == l->l_cpu->ci_schedstate.spc_lwplock);
lwp_lock(l);
+ l->l_stat = LSIDL;
+ spc_lock(l->l_cpu);
mi_switch(l);
KASSERT(curlwp == l);
KASSERT(l->l_stat == LSONPROC);
Index: src/sys/kern/kern_lwp.c
diff -u src/sys/kern/kern_lwp.c:1.216 src/sys/kern/kern_lwp.c:1.217
--- src/sys/kern/kern_lwp.c:1.216 Tue Dec 3 05:07:48 2019
+++ src/sys/kern/kern_lwp.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_lwp.c,v 1.216 2019/12/03 05:07:48 riastradh Exp $ */
+/* $NetBSD: kern_lwp.c,v 1.217 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -209,7 +209,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.216 2019/12/03 05:07:48 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.217 2019/12/06 21:36:10 ad Exp $");
#include "opt_ddb.h"
#include "opt_lockdebug.h"
@@ -1602,6 +1602,7 @@ lwp_userret(struct lwp *l)
lwp_unlock(l);
mutex_exit(p->p_lock);
lwp_lock(l);
+ spc_lock(l->l_cpu);
mi_switch(l);
}
Index: src/sys/kern/kern_sig.c
diff -u src/sys/kern/kern_sig.c:1.380 src/sys/kern/kern_sig.c:1.381
--- src/sys/kern/kern_sig.c:1.380 Thu Nov 21 18:17:36 2019
+++ src/sys/kern/kern_sig.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_sig.c,v 1.380 2019/11/21 18:17:36 ad Exp $ */
+/* $NetBSD: kern_sig.c,v 1.381 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -70,7 +70,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_sig.c,v 1.380 2019/11/21 18:17:36 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_sig.c,v 1.381 2019/12/06 21:36:10 ad Exp $");
#include "opt_ptrace.h"
#include "opt_dtrace.h"
@@ -1771,6 +1771,7 @@ sigswitch_unlock_and_switch_away(struct
mutex_exit(p->p_lock);
lwp_lock(l);
+ spc_lock(l->l_cpu);
mi_switch(l);
KERNEL_LOCK(biglocks, l);
}
Index: src/sys/kern/kern_sleepq.c
diff -u src/sys/kern/kern_sleepq.c:1.53 src/sys/kern/kern_sleepq.c:1.54
--- src/sys/kern/kern_sleepq.c:1.53 Sat Nov 23 19:42:52 2019
+++ src/sys/kern/kern_sleepq.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_sleepq.c,v 1.53 2019/11/23 19:42:52 ad Exp $ */
+/* $NetBSD: kern_sleepq.c,v 1.54 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.53 2019/11/23 19:42:52 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.54 2019/12/06 21:36:10 ad Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -260,6 +260,7 @@ sleepq_block(int timo, bool catch_p)
if (timo) {
callout_schedule(&l->l_timeout_ch, timo);
}
+ spc_lock(l->l_cpu);
mi_switch(l);
/* The LWP and sleep queue are now unlocked. */
Index: src/sys/kern/kern_softint.c
diff -u src/sys/kern/kern_softint.c:1.54 src/sys/kern/kern_softint.c:1.55
--- src/sys/kern/kern_softint.c:1.54 Fri Dec 6 18:15:57 2019
+++ src/sys/kern/kern_softint.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_softint.c,v 1.54 2019/12/06 18:15:57 ad Exp $ */
+/* $NetBSD: kern_softint.c,v 1.55 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -170,7 +170,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.54 2019/12/06 18:15:57 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.55 2019/12/06 21:36:10 ad Exp $");
#include <sys/param.h>
#include <sys/proc.h>
@@ -729,6 +729,7 @@ softint_thread(void *cookie)
lwp_lock(l);
l->l_stat = LSIDL;
+ spc_lock(l->l_cpu);
mi_switch(l);
}
}
Index: src/sys/kern/kern_synch.c
diff -u src/sys/kern/kern_synch.c:1.328 src/sys/kern/kern_synch.c:1.329
--- src/sys/kern/kern_synch.c:1.328 Tue Dec 3 05:07:48 2019
+++ src/sys/kern/kern_synch.c Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_synch.c,v 1.328 2019/12/03 05:07:48 riastradh Exp $ */
+/* $NetBSD: kern_synch.c,v 1.329 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019
@@ -69,7 +69,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.328 2019/12/03 05:07:48 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.329 2019/12/06 21:36:10 ad Exp $");
#include "opt_kstack.h"
#include "opt_dtrace.h"
@@ -269,11 +269,14 @@ yield(void)
KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
lwp_lock(l);
+
KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
KASSERT(l->l_stat == LSONPROC);
+
/* Voluntary - ditch kpriority boost. */
l->l_kpriority = false;
- (void)mi_switch(l);
+ spc_lock(l->l_cpu);
+ mi_switch(l);
KERNEL_LOCK(l->l_biglocks, l);
}
@@ -288,11 +291,14 @@ preempt(void)
KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
lwp_lock(l);
+
KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
KASSERT(l->l_stat == LSONPROC);
+
/* Involuntary - keep kpriority boost. */
l->l_pflag |= LP_PREEMPTING;
- (void)mi_switch(l);
+ spc_lock(l->l_cpu);
+ mi_switch(l);
KERNEL_LOCK(l->l_biglocks, l);
}
@@ -372,7 +378,9 @@ kpreempt(uintptr_t where)
kpreempt_ev_immed.ev_count++;
}
lwp_lock(l);
+ /* Involuntary - keep kpriority boost. */
l->l_pflag |= LP_PREEMPTING;
+ spc_lock(l->l_cpu);
mi_switch(l);
l->l_nopreempt++;
splx(s);
@@ -501,20 +509,22 @@ nextlwp(struct cpu_info *ci, struct sche
/*
* The machine independent parts of context switch.
*
- * Returns 1 if another LWP was actually run.
+ * NOTE: do not use l->l_cpu in this routine. The caller may have enqueued
+ * itself onto another CPU's run queue, so l->l_cpu may point elsewhere.
*/
-int
+void
mi_switch(lwp_t *l)
{
struct cpu_info *ci;
struct schedstate_percpu *spc;
struct lwp *newl;
- int retval, oldspl;
+ int oldspl;
struct bintime bt;
bool returning;
KASSERT(lwp_locked(l, NULL));
KASSERT(kpreempt_disabled());
+ KASSERT(mutex_owned(curcpu()->ci_schedstate.spc_mutex));
LOCKDEBUG_BARRIER(l->l_mutex, 1);
kstack_check_magic(l);
@@ -523,8 +533,8 @@ mi_switch(lwp_t *l)
KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp);
KASSERT((l->l_pflag & LP_RUNNING) != 0);
- KASSERT(l->l_cpu == curcpu());
- ci = l->l_cpu;
+ KASSERT(l->l_cpu == curcpu() || l->l_stat == LSRUN);
+ ci = curcpu();
spc = &ci->ci_schedstate;
returning = false;
newl = NULL;
@@ -555,31 +565,24 @@ mi_switch(lwp_t *l)
}
#endif /* !__HAVE_FAST_SOFTINTS */
- /* Lock the runqueue */
- KASSERT(l->l_stat != LSRUN);
- mutex_spin_enter(spc->spc_mutex);
-
/*
* If on the CPU and we have gotten this far, then we must yield.
*/
if (l->l_stat == LSONPROC && l != newl) {
KASSERT(lwp_locked(l, spc->spc_lwplock));
- if ((l->l_flag & LW_IDLE) == 0) {
- l->l_stat = LSRUN;
- lwp_setlock(l, spc->spc_mutex);
- sched_enqueue(l);
- /*
- * Handle migration. Note that "migrating LWP" may
- * be reset here, if interrupt/preemption happens
- * early in idle LWP.
- */
- if (l->l_target_cpu != NULL &&
- (l->l_pflag & LP_BOUND) == 0) {
- KASSERT((l->l_pflag & LP_INTR) == 0);
- spc->spc_migrating = l;
- }
- } else
- l->l_stat = LSIDL;
+ KASSERT((l->l_flag & LW_IDLE) == 0);
+ l->l_stat = LSRUN;
+ lwp_setlock(l, spc->spc_mutex);
+ sched_enqueue(l);
+ /*
+ * Handle migration. Note that "migrating LWP" may
+ * be reset here, if interrupt/preemption happens
+ * early in idle LWP.
+ */
+ if (l->l_target_cpu != NULL && (l->l_pflag & LP_BOUND) == 0) {
+ KASSERT((l->l_pflag & LP_INTR) == 0);
+ spc->spc_migrating = l;
+ }
}
/* Pick new LWP to run. */
@@ -694,6 +697,7 @@ mi_switch(lwp_t *l)
while (newl->l_ctxswtch)
SPINLOCK_BACKOFF(count);
}
+ membar_enter();
/*
* If DTrace has set the active vtime enum to anything
@@ -743,28 +747,25 @@ mi_switch(lwp_t *l)
l->l_lwpctl->lc_pctr++;
}
- KASSERT(l->l_cpu == ci);
- splx(oldspl);
/*
- * note that, unless the caller disabled preemption,
- * we can be preempted at any time after the above splx() call.
+ * Note that, unless the caller disabled preemption, we can
+ * be preempted at any time after this splx().
*/
- retval = 1;
+ splx(oldspl);
} else {
/* Nothing to do - just unlock and return. */
mutex_spin_exit(spc->spc_mutex);
l->l_pflag &= ~LP_PREEMPTING;
lwp_unlock(l);
- retval = 0;
}
+ /* Only now is it safe to consider l_cpu again. */
KASSERT(l == curlwp);
+ KASSERT(l->l_cpu == ci);
KASSERT(l->l_stat == LSONPROC);
SYSCALL_TIME_WAKEUP(l);
LOCKDEBUG_BARRIER(NULL, 1);
-
- return retval;
}
/*
@@ -848,6 +849,7 @@ lwp_exit_switchaway(lwp_t *l)
while (newl->l_ctxswtch)
SPINLOCK_BACKOFF(count);
}
+ membar_enter();
/*
* If DTrace has set the active vtime enum to anything
Index: src/sys/sys/sched.h
diff -u src/sys/sys/sched.h:1.79 src/sys/sys/sched.h:1.80
--- src/sys/sys/sched.h:1.79 Tue Dec 3 22:28:41 2019
+++ src/sys/sys/sched.h Fri Dec 6 21:36:10 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: sched.h,v 1.79 2019/12/03 22:28:41 ad Exp $ */
+/* $NetBSD: sched.h,v 1.80 2019/12/06 21:36:10 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2007, 2008, 2019
@@ -266,7 +266,7 @@ void sched_print_runqueue(void (*pr)(co
bool kpreempt(uintptr_t);
void preempt(void);
void yield(void);
-int mi_switch(struct lwp *);
+void mi_switch(struct lwp *);
void updatertime(lwp_t *, const struct bintime *);
void sched_idle(void);
void suspendsched(void);
