Module Name: src
Committed By: bouyer
Date: Sat Apr 11 18:26:08 UTC 2020
Modified Files:
src/sys/arch/amd64/amd64 [bouyer-xenpvh]: lock_stubs.S spl.S
src/sys/arch/i386/i386 [bouyer-xenpvh]: spl.S
src/sys/arch/x86/conf [bouyer-xenpvh]: files.x86
src/sys/arch/x86/include [bouyer-xenpvh]: intr.h
src/sys/arch/x86/x86 [bouyer-xenpvh]: intr.c x86_machdep.c
src/sys/arch/xen/conf [bouyer-xenpvh]: files.xen
src/sys/arch/xen/include [bouyer-xenpvh]: intrdefs.h
src/sys/arch/xen/x86 [bouyer-xenpvh]: xen_ipi.c xenfunc.c
src/sys/arch/xen/xen [bouyer-xenpvh]: clock.c
Added Files:
src/sys/arch/x86/x86 [bouyer-xenpvh]: x86_softintr.c
Log Message:
Move softint and preemtion-related functions out of x86/x86/intr.c to
its own file, x86/x86/x86_softintr.c
Add x86/x86/x86_softintr.c for native and XenPV
Make sure XenPV also check ci_ioending, which is used for softints.
Switch XenPV to fast softints and allow kernel preemption.
kpreempt_disable() before calling pmap_changeprot_local()
run xen_wallclock_time() and xen_global_systime_ns() at splshed() to
avoid being interrupted.
XXX amd64 lock stubs are racy for XPENDING
To generate a diff of this commit:
cvs rdiff -u -r1.35 -r1.35.6.1 src/sys/arch/amd64/amd64/lock_stubs.S
cvs rdiff -u -r1.43.4.4 -r1.43.4.5 src/sys/arch/amd64/amd64/spl.S
cvs rdiff -u -r1.50.4.3 -r1.50.4.4 src/sys/arch/i386/i386/spl.S
cvs rdiff -u -r1.107 -r1.107.10.1 src/sys/arch/x86/conf/files.x86
cvs rdiff -u -r1.61.6.1 -r1.61.6.2 src/sys/arch/x86/include/intr.h
cvs rdiff -u -r1.150 -r1.150.6.1 src/sys/arch/x86/x86/intr.c
cvs rdiff -u -r1.137.2.2 -r1.137.2.3 src/sys/arch/x86/x86/x86_machdep.c
cvs rdiff -u -r0 -r1.1.2.1 src/sys/arch/x86/x86/x86_softintr.c
cvs rdiff -u -r1.180 -r1.180.2.1 src/sys/arch/xen/conf/files.xen
cvs rdiff -u -r1.15 -r1.15.2.1 src/sys/arch/xen/include/intrdefs.h
cvs rdiff -u -r1.35 -r1.35.6.1 src/sys/arch/xen/x86/xen_ipi.c
cvs rdiff -u -r1.26 -r1.26.8.1 src/sys/arch/xen/x86/xenfunc.c
cvs rdiff -u -r1.80 -r1.80.6.1 src/sys/arch/xen/xen/clock.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/lock_stubs.S
diff -u src/sys/arch/amd64/amd64/lock_stubs.S:1.35 src/sys/arch/amd64/amd64/lock_stubs.S:1.35.6.1
--- src/sys/arch/amd64/amd64/lock_stubs.S:1.35 Sun Dec 8 20:00:56 2019
+++ src/sys/arch/amd64/amd64/lock_stubs.S Sat Apr 11 18:26:06 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs.S,v 1.35 2019/12/08 20:00:56 ad Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.35.6.1 2020/04/11 18:26:06 bouyer Exp $ */
/*
* Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -126,15 +126,12 @@ ENTRY(mutex_spin_exit)
jnz 1f
cmpl CPU_INFO_ILEVEL(%r8), %edi
jae 1f
-#if !defined(XENPV)
movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi
CLI(ax)
testl CPU_INFO_IPENDING(%r8), %esi
jnz _C_LABEL(Xspllower)
-#endif
#if defined(XEN)
movl CPU_INFO_XUNMASK(%r8,%rdi,4), %esi
- CLI(ax)
testl CPU_INFO_XPENDING(%r8), %esi
jnz _C_LABEL(Xspllower)
#endif
@@ -155,14 +152,12 @@ ENTRY(mutex_spin_exit)
cmpl %edx,%ecx /* new level is lower? */
jae 2f
1:
-#if !defined(XENPV)
movl CPU_INFO_IPENDING(%rsi),%eax
testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
jnz 3f
movl %eax,%ebx
cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */
jnz 4f
-#endif
#if defined(XEN)
movl CPU_INFO_XPENDING(%rsi),%eax
testl %eax,CPU_INFO_XUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
Index: src/sys/arch/amd64/amd64/spl.S
diff -u src/sys/arch/amd64/amd64/spl.S:1.43.4.4 src/sys/arch/amd64/amd64/spl.S:1.43.4.5
--- src/sys/arch/amd64/amd64/spl.S:1.43.4.4 Sat Apr 11 10:11:30 2020
+++ src/sys/arch/amd64/amd64/spl.S Sat Apr 11 18:26:06 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: spl.S,v 1.43.4.4 2020/04/11 10:11:30 bouyer Exp $ */
+/* $NetBSD: spl.S,v 1.43.4.5 2020/04/11 18:26:06 bouyer Exp $ */
/*
* Copyright (c) 2003 Wasabi Systems, Inc.
@@ -91,7 +91,6 @@ ENTRY(splraise)
ret
END(splraise)
-#ifndef XENPV
/*
* Xsoftintr()
*
@@ -148,11 +147,11 @@ IDTVEC(softintr)
movq PCB_RSP0(%rdx),%rsp
/* dispatch */
- sti
+ STI(di)
movq %r15,%rdi /* interrupted LWP */
movl IS_MAXLEVEL(%rax),%esi /* ipl to run at */
call _C_LABEL(softint_dispatch)/* run handlers */
- cli
+ CLI(di)
/* restore old context */
movq L_PCB(%r15),%rcx
@@ -174,7 +173,7 @@ IDTVEC_END(softintr)
*/
ENTRY(softintr_ret)
incl CPUVAR(MTX_COUNT) /* re-adjust after mi_switch */
- cli
+ CLI(ax) /* %rax not used by Xspllower/Xdoreti */
jmp *%r13 /* back to Xspllower/Xdoreti */
END(softintr_ret)
@@ -196,11 +195,11 @@ END(softint_trigger)
*/
IDTVEC(recurse_preempt)
movl $IPL_PREEMPT,CPUVAR(ILEVEL)
- sti
+ STI(di)
xorq %rdi,%rdi
KMSAN_INIT_ARG(8)
call _C_LABEL(kpreempt)
- cli
+ CLI(di)
jmp *%r13 /* back to Xspllower */
IDTVEC_END(recurse_preempt)
@@ -211,20 +210,19 @@ IDTVEC_END(recurse_preempt)
*/
IDTVEC(resume_preempt)
movl $IPL_PREEMPT,CPUVAR(ILEVEL)
- sti
+ STI(ax)
testq $SEL_RPL,TF_CS(%rsp)
jnz 1f
movq TF_RIP(%rsp),%rdi
KMSAN_INIT_ARG(8)
call _C_LABEL(kpreempt) /* from kernel */
- cli
+ CLI(ax)
jmp *%r13 /* back to Xdoreti */
1:
call _C_LABEL(preempt) /* from user */
- cli
+ CLI(ax)
jmp *%r13 /* back to Xdoreti */
IDTVEC_END(resume_preempt)
-#endif /* XEN */
/*
* void spllower(int s);
@@ -336,7 +334,6 @@ IDTVEC(spllower)
movl %edi,%ebx
leaq 1f(%rip),%r13 /* address to resume loop at */
1:
-#if !defined(XENPV)
movl %ebx,%eax /* get cpl */
movl CPUVAR(IUNMASK)(,%rax,4),%eax
CLI(si)
@@ -346,7 +343,6 @@ IDTVEC(spllower)
btrl %eax,CPUVAR(IPENDING)
movq CPUVAR(ISOURCES)(,%rax,8),%rax
jmp *IS_RECURSE(%rax)
-#endif
2:
#if defined(XEN)
movl %ebx,%eax /* get cpl */
@@ -382,7 +378,6 @@ IDTVEC(doreti)
decl CPUVAR(IDEPTH)
leaq 1f(%rip),%r13
1:
-#if !defined(XENPV)
movl %ebx,%eax
movl CPUVAR(IUNMASK)(,%rax,4),%eax
CLI(si)
@@ -392,7 +387,6 @@ IDTVEC(doreti)
btrl %eax,CPUVAR(IPENDING)
movq CPUVAR(ISOURCES)(,%rax,8),%rax
jmp *IS_RESUME(%rax)
-#endif
2:
#if defined(XEN)
movl %ebx,%eax
Index: src/sys/arch/i386/i386/spl.S
diff -u src/sys/arch/i386/i386/spl.S:1.50.4.3 src/sys/arch/i386/i386/spl.S:1.50.4.4
--- src/sys/arch/i386/i386/spl.S:1.50.4.3 Sat Apr 11 12:01:42 2020
+++ src/sys/arch/i386/i386/spl.S Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: spl.S,v 1.50.4.3 2020/04/11 12:01:42 bouyer Exp $ */
+/* $NetBSD: spl.S,v 1.50.4.4 2020/04/11 18:26:07 bouyer Exp $ */
/*
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.50.4.3 2020/04/11 12:01:42 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.50.4.4 2020/04/11 18:26:07 bouyer Exp $");
#include "opt_ddb.h"
#include "opt_spldebug.h"
@@ -213,7 +213,6 @@ IDTVEC(spllower)
jz .Lspllower_panic
#endif /* XENPV */
#endif /* defined(DEBUG) */
-#if !defined(XENPV)
movl %ebx,%eax /* get cpl */
movl CPUVAR(IUNMASK)(,%eax,4),%eax
andl CPUVAR(IPENDING),%eax /* any non-masked bits left? */
@@ -222,7 +221,6 @@ IDTVEC(spllower)
btrl %eax,CPUVAR(IPENDING)
movl CPUVAR(ISOURCES)(,%eax,4),%eax
jmp *IS_RECURSE(%eax)
-#endif
2:
#if defined(XEN)
movl %ebx,%eax /* get cpl */
@@ -361,7 +359,6 @@ END(doreti_checkast)
#endif
IDTVEC_END(doreti)
-#ifndef XENPV
/*
* Xsoftintr()
*
@@ -379,6 +376,7 @@ IDTVEC(softintr)
pushl %esi
pushl %edi
movl $IPL_HIGH,CPUVAR(ILEVEL)
+ STI(%esi)
movl CPUVAR(CURLWP),%esi
movl IS_LWP(%eax),%edi /* switch to handler LWP */
movl %edi,CPUVAR(CURLWP)
@@ -387,12 +385,11 @@ IDTVEC(softintr)
movl %esp,PCB_ESP(%ecx)
movl %ebp,PCB_EBP(%ecx)
movl PCB_ESP0(%edx),%esp /* onto new stack */
- sti
pushl IS_MAXLEVEL(%eax) /* ipl to run at */
pushl %esi
call _C_LABEL(softint_dispatch)/* run handlers */
addl $8,%esp
- cli
+ CLI(%ecx)
movl L_PCB(%esi),%ecx
movl PCB_ESP(%ecx),%esp
xchgl %esi,CPUVAR(CURLWP) /* must be globally visible */
@@ -412,7 +409,7 @@ IDTVEC_END(softintr)
*/
ENTRY(softintr_ret)
incl CPUVAR(MTX_COUNT) /* re-adjust after mi_switch */
- cli
+ CLI(%eax)
jmp *%esi /* back to splx/doreti */
END(softintr_ret)
@@ -434,11 +431,11 @@ END(softint_trigger)
*/
IDTVEC(recurse_preempt)
movl $IPL_PREEMPT,CPUVAR(ILEVEL)
- sti
+ STI(%eax)
pushl $0
call _C_LABEL(kpreempt)
addl $4,%esp
- cli
+ CLI(%eax)
jmp *%esi
IDTVEC_END(recurse_preempt)
@@ -449,18 +446,17 @@ IDTVEC_END(recurse_preempt)
*/
IDTVEC(resume_preempt)
movl $IPL_PREEMPT,CPUVAR(ILEVEL)
- sti
+ STI(%eax)
testb $CHK_UPL,TF_CS(%esp)
jnz 1f
movl TF_EIP(%esp),%eax
pushl %eax
call _C_LABEL(kpreempt) /* from kernel */
addl $4,%esp
- cli
+ CLI(%eax)
jmp *%esi
1:
call _C_LABEL(preempt) /* from user */
- cli
+ CLI(%eax)
jmp *%esi
IDTVEC_END(resume_preempt)
-#endif /* !XENPV */
Index: src/sys/arch/x86/conf/files.x86
diff -u src/sys/arch/x86/conf/files.x86:1.107 src/sys/arch/x86/conf/files.x86:1.107.10.1
--- src/sys/arch/x86/conf/files.x86:1.107 Fri Feb 15 08:54:01 2019
+++ src/sys/arch/x86/conf/files.x86 Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.x86,v 1.107 2019/02/15 08:54:01 nonaka Exp $
+# $NetBSD: files.x86,v 1.107.10.1 2020/04/11 18:26:07 bouyer Exp $
# options for MP configuration through the MP spec
defflag opt_mpbios.h MPBIOS MPDEBUG MPBIOS_SCANPCI
@@ -93,6 +93,7 @@ file arch/x86/x86/genfb_machdep.c machde
file arch/x86/x86/identcpu.c machdep
file arch/x86/x86/i8259.c machdep
file arch/x86/x86/intr.c machdep
+file arch/x86/x86/x86_softintr.c machdep
file arch/x86/x86/kgdb_machdep.c kgdb
file arch/x86/x86/nmi.c machdep
file arch/x86/x86/idt.c machdep
Index: src/sys/arch/x86/include/intr.h
diff -u src/sys/arch/x86/include/intr.h:1.61.6.1 src/sys/arch/x86/include/intr.h:1.61.6.2
--- src/sys/arch/x86/include/intr.h:1.61.6.1 Fri Apr 10 14:42:00 2020
+++ src/sys/arch/x86/include/intr.h Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.h,v 1.61.6.1 2020/04/10 14:42:00 bouyer Exp $ */
+/* $NetBSD: intr.h,v 1.61.6.2 2020/04/11 18:26:07 bouyer Exp $ */
/*-
* Copyright (c) 1998, 2001, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -32,12 +32,10 @@
#ifndef _X86_INTR_H_
#define _X86_INTR_H_
-#if !defined(XENPV)
#define __HAVE_FAST_SOFTINTS
#if !defined(NO_PREEMPTION)
#define __HAVE_PREEMPTION
#endif /* !defined(NO_PREEMPTION) */
-#endif /* !defined(XENPV) */
#ifdef _KERNEL
#include <sys/types.h>
@@ -239,6 +237,9 @@ const char *intr_create_intrid(int, stru
struct intrsource *intr_allocate_io_intrsource(const char *);
void intr_free_io_intrsource(const char *);
+void x86_init_preempt(struct cpu_info *);
+void x86_intr_calculatemasks(struct cpu_info *);
+
int x86_send_ipi(struct cpu_info *, int);
void x86_broadcast_ipi(int);
void x86_ipi_handler(void);
Index: src/sys/arch/x86/x86/intr.c
diff -u src/sys/arch/x86/x86/intr.c:1.150 src/sys/arch/x86/x86/intr.c:1.150.6.1
--- src/sys/arch/x86/x86/intr.c:1.150 Mon Dec 30 23:32:30 2019
+++ src/sys/arch/x86/x86/intr.c Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.c,v 1.150 2019/12/30 23:32:30 thorpej Exp $ */
+/* $NetBSD: intr.c,v 1.150.6.1 2020/04/11 18:26:07 bouyer Exp $ */
/*
* Copyright (c) 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -133,7 +133,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.150 2019/12/30 23:32:30 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.150.6.1 2020/04/11 18:26:07 bouyer Exp $");
#include "opt_intrdebug.h"
#include "opt_multiprocessor.h"
@@ -207,16 +207,6 @@ extern void Xrecurse_hyperv_hypercall(vo
#define DPRINTF(msg)
#endif
-struct pic softintr_pic = {
- .pic_name = "softintr_fakepic",
- .pic_type = PIC_SOFT,
- .pic_vecbase = 0,
- .pic_apicid = 0,
- .pic_lock = __SIMPLELOCK_UNLOCKED,
-};
-
-static void intr_calculatemasks(struct cpu_info *);
-
static SIMPLEQ_HEAD(, intrsource) io_interrupt_sources =
SIMPLEQ_HEAD_INITIALIZER(io_interrupt_sources);
@@ -290,73 +280,6 @@ x86_nmi(void)
}
/*
- * Recalculate the interrupt masks from scratch.
- * During early boot, anything goes and we are always called on the BP.
- * When the system is up and running:
- *
- * => called with ci == curcpu()
- * => cpu_lock held by the initiator
- * => interrupts disabled on-chip (PSL_I)
- *
- * Do not call printf(), kmem_free() or other "heavyweight" routines
- * from here. This routine must be quick and must not block.
- */
-static void
-intr_calculatemasks(struct cpu_info *ci)
-{
- int irq, level, unusedirqs, intrlevel[MAX_INTR_SOURCES];
- struct intrhand *q;
-
- /* First, figure out which levels each IRQ uses. */
- unusedirqs = 0xffffffff;
- for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
- int levels = 0;
-
- if (ci->ci_isources[irq] == NULL) {
- intrlevel[irq] = 0;
- continue;
- }
- for (q = ci->ci_isources[irq]->is_handlers; q; q = q->ih_next)
- levels |= 1U << q->ih_level;
- intrlevel[irq] = levels;
- if (levels)
- unusedirqs &= ~(1U << irq);
- }
-
- /* Then figure out which IRQs use each level. */
- for (level = 0; level < NIPL; level++) {
- int irqs = 0;
- for (irq = 0; irq < MAX_INTR_SOURCES; irq++)
- if (intrlevel[irq] & (1U << level))
- irqs |= 1U << irq;
- ci->ci_imask[level] = irqs | unusedirqs;
- }
-
- for (level = 0; level<(NIPL-1); level++)
- ci->ci_imask[level+1] |= ci->ci_imask[level];
-
- for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
- int maxlevel = IPL_NONE;
- int minlevel = IPL_HIGH;
-
- if (ci->ci_isources[irq] == NULL)
- continue;
- for (q = ci->ci_isources[irq]->is_handlers; q;
- q = q->ih_next) {
- if (q->ih_level < minlevel)
- minlevel = q->ih_level;
- if (q->ih_level > maxlevel)
- maxlevel = q->ih_level;
- }
- ci->ci_isources[irq]->is_maxlevel = maxlevel;
- ci->ci_isources[irq]->is_minlevel = minlevel;
- }
-
- for (level = 0; level < NIPL; level++)
- ci->ci_iunmask[level] = ~ci->ci_imask[level];
-}
-
-/*
* Create an interrupt id such as "ioapic0 pin 9". This interrupt id is used
* by MI code and intrctl(8).
*/
@@ -801,7 +724,7 @@ intr_establish_xcall(void *arg1, void *a
/* Link in the handler and re-calculate masks. */
*(ih->ih_prevp) = ih;
- intr_calculatemasks(ci);
+ x86_intr_calculatemasks(ci);
/* Hook in new IDT vector and SPL state. */
if (source->is_resume == NULL || source->is_idtvec != idt_vec) {
@@ -1186,7 +1109,7 @@ intr_disestablish_xcall(void *arg1, void
*p = q->ih_next;
- intr_calculatemasks(ci);
+ x86_intr_calculatemasks(ci);
/*
* If there is no any handler, 1) do delroute because it has no
* any source and 2) dont' hwunmask to prevent spurious interrupt.
@@ -1327,15 +1250,10 @@ intr_string(intr_handle_t ih, char *buf,
/*
* Fake interrupt handler structures for the benefit of symmetry with
- * other interrupt sources, and the benefit of intr_calculatemasks()
+ * other interrupt sources, and the benefit of x86_intr_calculatemasks()
*/
-struct intrhand fake_softclock_intrhand;
-struct intrhand fake_softnet_intrhand;
-struct intrhand fake_softserial_intrhand;
-struct intrhand fake_softbio_intrhand;
struct intrhand fake_timer_intrhand;
struct intrhand fake_ipi_intrhand;
-struct intrhand fake_preempt_intrhand;
#if NHYPERV > 0
struct intrhand fake_hyperv_intrhand;
#endif
@@ -1369,7 +1287,7 @@ redzone_const_or_zero(int x)
void
cpu_intr_init(struct cpu_info *ci)
{
-#if (NLAPIC > 0) || defined(MULTIPROCESSOR) || defined(__HAVE_PREEMPTION) || \
+#if (NLAPIC > 0) || defined(MULTIPROCESSOR) || \
(NHYPERV > 0)
struct intrsource *isp;
#endif
@@ -1423,16 +1341,10 @@ cpu_intr_init(struct cpu_info *ci)
#endif
#if defined(__HAVE_PREEMPTION)
- isp = kmem_zalloc(sizeof(*isp), KM_SLEEP);
- isp->is_recurse = Xrecurse_preempt;
- isp->is_resume = Xresume_preempt;
- fake_preempt_intrhand.ih_level = IPL_PREEMPT;
- isp->is_handlers = &fake_preempt_intrhand;
- isp->is_pic = &softintr_pic;
- ci->ci_isources[SIR_PREEMPT] = isp;
+ x86_init_preempt(ci);
#endif
- intr_calculatemasks(ci);
+ x86_intr_calculatemasks(ci);
#if defined(INTRSTACKSIZE)
vaddr_t istack;
@@ -1521,55 +1433,6 @@ intr_printconfig(void)
#endif
-#if defined(__HAVE_FAST_SOFTINTS)
-void
-softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
-{
- struct intrsource *isp;
- struct cpu_info *ci;
- u_int sir;
-
- ci = l->l_cpu;
-
- isp = kmem_zalloc(sizeof(*isp), KM_SLEEP);
- isp->is_recurse = Xsoftintr;
- isp->is_resume = Xsoftintr;
- isp->is_pic = &softintr_pic;
-
- switch (level) {
- case SOFTINT_BIO:
- sir = SIR_BIO;
- fake_softbio_intrhand.ih_level = IPL_SOFTBIO;
- isp->is_handlers = &fake_softbio_intrhand;
- break;
- case SOFTINT_NET:
- sir = SIR_NET;
- fake_softnet_intrhand.ih_level = IPL_SOFTNET;
- isp->is_handlers = &fake_softnet_intrhand;
- break;
- case SOFTINT_SERIAL:
- sir = SIR_SERIAL;
- fake_softserial_intrhand.ih_level = IPL_SOFTSERIAL;
- isp->is_handlers = &fake_softserial_intrhand;
- break;
- case SOFTINT_CLOCK:
- sir = SIR_CLOCK;
- fake_softclock_intrhand.ih_level = IPL_SOFTCLOCK;
- isp->is_handlers = &fake_softclock_intrhand;
- break;
- default:
- panic("softint_init_md");
- }
-
- KASSERT(ci->ci_isources[sir] == NULL);
-
- *machdep = (1 << sir);
- ci->ci_isources[sir] = isp;
- ci->ci_isources[sir]->is_lwp = l;
-
- intr_calculatemasks(ci);
-}
-#endif /* __HAVE_FAST_SOFTINTS */
/*
* Save current affinitied cpu's interrupt count.
*/
@@ -1628,7 +1491,7 @@ intr_redistribute_xc_t(void *arg1, void
/* Hook it in and re-calculate masks. */
ci->ci_isources[slot] = isp;
- intr_calculatemasks(curcpu());
+ x86_intr_calculatemasks(curcpu());
/* Re-enable interrupts locally. */
x86_write_psl(psl);
@@ -1682,7 +1545,7 @@ intr_redistribute_xc_s2(void *arg1, void
/* Patch out the source and re-calculate masks. */
ci->ci_isources[slot] = NULL;
- intr_calculatemasks(ci);
+ x86_intr_calculatemasks(ci);
/* Re-enable interrupts locally. */
x86_write_psl(psl);
@@ -1872,7 +1735,7 @@ intr_activate_xcall(void *arg1, void *ar
psl = x86_read_psl();
x86_disable_intr();
- intr_calculatemasks(ci);
+ x86_intr_calculatemasks(ci);
if (source->is_type == IST_LEVEL) {
stubp = &source->is_pic->pic_level_stubs[slot];
@@ -1917,7 +1780,7 @@ intr_deactivate_xcall(void *arg1, void *
ci->ci_nintrhand--;
}
- intr_calculatemasks(ci);
+ x86_intr_calculatemasks(ci);
/*
* Skip unsetgate(), because the same itd[] entry is overwritten in
Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.137.2.2 src/sys/arch/x86/x86/x86_machdep.c:1.137.2.3
--- src/sys/arch/x86/x86/x86_machdep.c:1.137.2.2 Sat Apr 11 08:06:16 2020
+++ src/sys/arch/x86/x86/x86_machdep.c Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.137.2.2 2020/04/11 08:06:16 bouyer Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.137.2.3 2020/04/11 18:26:07 bouyer Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.137.2.2 2020/04/11 08:06:16 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.137.2.3 2020/04/11 18:26:07 bouyer Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -307,7 +307,11 @@ cpu_need_resched(struct cpu_info *ci, st
#ifdef __HAVE_PREEMPTION
if ((flags & RESCHED_KPREEMPT) != 0) {
if ((flags & RESCHED_REMOTE) != 0) {
+#ifdef XENPV
+ xen_send_ipi(ci, XEN_IPI_KPREEMPT);
+#else
x86_send_ipi(ci, X86_IPI_KPREEMPT);
+#endif
} else {
softint_trigger(1 << SIR_PREEMPT);
}
Index: src/sys/arch/xen/conf/files.xen
diff -u src/sys/arch/xen/conf/files.xen:1.180 src/sys/arch/xen/conf/files.xen:1.180.2.1
--- src/sys/arch/xen/conf/files.xen:1.180 Fri Apr 3 22:45:30 2020
+++ src/sys/arch/xen/conf/files.xen Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.xen,v 1.180 2020/04/03 22:45:30 ad Exp $
+# $NetBSD: files.xen,v 1.180.2.1 2020/04/11 18:26:07 bouyer Exp $
# NetBSD: files.x86,v 1.10 2003/10/08 17:30:00 bouyer Exp
# NetBSD: files.i386,v 1.254 2004/03/25 23:32:10 jmc Exp
@@ -148,6 +148,7 @@ file arch/xen/x86/pintr.c machdep & dom
file arch/xen/x86/xen_ipi.c multiprocessor & xenpv
file arch/x86/x86/idt.c machdep
file arch/x86/x86/intr.c machdep & xenpvhvm
+file arch/x86/x86/x86_softintr.c machdep
file arch/x86/x86/ipi.c xenpvhvm
file arch/x86/x86/pmap.c machdep
file arch/x86/x86/x86_tlb.c machdep
Index: src/sys/arch/xen/include/intrdefs.h
diff -u src/sys/arch/xen/include/intrdefs.h:1.15 src/sys/arch/xen/include/intrdefs.h:1.15.2.1
--- src/sys/arch/xen/include/intrdefs.h:1.15 Fri Apr 3 22:20:36 2020
+++ src/sys/arch/xen/include/intrdefs.h Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: intrdefs.h,v 1.15 2020/04/03 22:20:36 ad Exp $ */
+/* $NetBSD: intrdefs.h,v 1.15.2.1 2020/04/11 18:26:07 bouyer Exp $ */
/* This file co-exists, and is included via machine/intrdefs.h */
@@ -14,9 +14,10 @@
#define XEN_IPI_HVCB 0x00000010
#define XEN_IPI_GENERIC 0x00000020
#define XEN_IPI_AST 0x00000040
+#define XEN_IPI_KPREEMPT 0x00000080
/* Note: IPI_KICK does not have a handler. */
-#define XEN_NIPIS 7
+#define XEN_NIPIS 8
/* The number of 'irqs' that XEN understands */
#define NUM_XEN_IRQS 256
@@ -24,6 +25,6 @@
#define XEN_IPI_NAMES { "halt IPI", "FPU synch IPI", \
"DDB IPI", "xcall IPI", \
"HVCB IPI", "generic IPI", \
- "AST IPI" }
+ "AST IPI", "kpreempt IPI" }
#endif /* _XEN_INTRDEFS_H_ */
Index: src/sys/arch/xen/x86/xen_ipi.c
diff -u src/sys/arch/xen/x86/xen_ipi.c:1.35 src/sys/arch/xen/x86/xen_ipi.c:1.35.6.1
--- src/sys/arch/xen/x86/xen_ipi.c:1.35 Sun Dec 1 15:34:46 2019
+++ src/sys/arch/xen/x86/xen_ipi.c Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $ */
+/* $NetBSD: xen_ipi.c,v 1.35.6.1 2020/04/11 18:26:07 bouyer Exp $ */
/*-
* Copyright (c) 2011, 2019 The NetBSD Foundation, Inc.
@@ -33,10 +33,10 @@
/*
* Based on: x86/ipi.c
- * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $");
+ * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35.6.1 2020/04/11 18:26:07 bouyer Exp $");
*/
-__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35.6.1 2020/04/11 18:26:07 bouyer Exp $");
#include "opt_ddb.h"
@@ -72,6 +72,7 @@ static void xen_ipi_xcall(struct cpu_inf
static void xen_ipi_hvcb(struct cpu_info *, struct intrframe *);
static void xen_ipi_generic(struct cpu_info *, struct intrframe *);
static void xen_ipi_ast(struct cpu_info *, struct intrframe *);
+static void xen_ipi_kpreempt(struct cpu_info *ci, struct intrframe *);
static void (*ipifunc[XEN_NIPIS])(struct cpu_info *, struct intrframe *) =
{ /* In order of priority (see: xen/include/intrdefs.h */
@@ -85,7 +86,8 @@ static void (*ipifunc[XEN_NIPIS])(struct
xen_ipi_xcall,
xen_ipi_hvcb,
xen_ipi_generic,
- xen_ipi_ast
+ xen_ipi_ast,
+ xen_ipi_kpreempt
};
static int
@@ -342,3 +344,9 @@ xen_ipi_hvcb(struct cpu_info *ci, struct
hypervisor_force_callback();
}
+
+static void
+xen_ipi_kpreempt(struct cpu_info *ci, struct intrframe * intrf)
+{
+ softint_trigger(1 << SIR_PREEMPT);
+}
Index: src/sys/arch/xen/x86/xenfunc.c
diff -u src/sys/arch/xen/x86/xenfunc.c:1.26 src/sys/arch/xen/x86/xenfunc.c:1.26.8.1
--- src/sys/arch/xen/x86/xenfunc.c:1.26 Sat May 4 11:15:49 2019
+++ src/sys/arch/xen/x86/xenfunc.c Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: xenfunc.c,v 1.26 2019/05/04 11:15:49 kre Exp $ */
+/* $NetBSD: xenfunc.c,v 1.26.8.1 2020/04/11 18:26:07 bouyer Exp $ */
/*
* Copyright (c) 2004 Christian Limpach.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xenfunc.c,v 1.26 2019/05/04 11:15:49 kre Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xenfunc.c,v 1.26.8.1 2020/04/11 18:26:07 bouyer Exp $");
#include <sys/param.h>
@@ -61,6 +61,7 @@ lidt(struct region_descriptor *rd)
* will be available at the boot stage when this is called.
*/
static char xen_idt_page[PAGE_SIZE] __attribute__((__aligned__ (PAGE_SIZE)));
+ kpreempt_disable();
memset(xen_idt_page, 0, PAGE_SIZE);
struct trap_info *xen_idt = (void * )xen_idt_page;
@@ -96,6 +97,7 @@ lidt(struct region_descriptor *rd)
/* reset */
pmap_changeprot_local((vaddr_t) xen_idt, VM_PROT_READ|VM_PROT_WRITE);
#endif /* __x86_64 */
+ kpreempt_enable();
}
void
@@ -256,6 +258,12 @@ rcr2(void)
return curcpu()->ci_vcpu->arch.cr2;
}
+void
+lcr2(register_t v)
+{
+ curcpu()->ci_vcpu->arch.cr2 = v;
+}
+
#ifdef __x86_64__
void
setusergs(int gssel)
Index: src/sys/arch/xen/xen/clock.c
diff -u src/sys/arch/xen/xen/clock.c:1.80 src/sys/arch/xen/xen/clock.c:1.80.6.1
--- src/sys/arch/xen/xen/clock.c:1.80 Wed Oct 16 18:29:49 2019
+++ src/sys/arch/xen/xen/clock.c Sat Apr 11 18:26:07 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: clock.c,v 1.80 2019/10/16 18:29:49 christos Exp $ */
+/* $NetBSD: clock.c,v 1.80.6.1 2020/04/11 18:26:07 bouyer Exp $ */
/*-
* Copyright (c) 2017, 2018 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
#endif
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.80 2019/10/16 18:29:49 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.80.6.1 2020/04/11 18:26:07 bouyer Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -501,6 +501,7 @@ xen_wallclock_time(struct timespec *tsp)
struct xen_wallclock_ticket ticket;
uint64_t systime_ns;
+ int s = splsched(); /* make sure we won't be interrupted */
/* Read the last wall clock sample from the hypervisor. */
do {
xen_wallclock_enter(&ticket);
@@ -510,6 +511,7 @@ xen_wallclock_time(struct timespec *tsp)
/* Get the global system time. */
systime_ns = xen_global_systime_ns();
+ splx(s);
/* Add the system time to the wall clock time. */
systime_ns += tsp->tv_nsec;
@@ -530,7 +532,6 @@ xen_global_systime_ns(void)
{
struct cpu_info *ci;
uint64_t local, global, result;
- int bound;
/*
* Find the local timecount on this CPU, and make sure it does
@@ -540,7 +541,7 @@ xen_global_systime_ns(void)
*
* XXX Can we avoid retrying if the CAS fails?
*/
- bound = curlwp_bind();
+ int s = splsched(); /* make sure we won't be interrupted */
ci = curcpu();
do {
local = xen_vcputime_systime_ns();
@@ -555,7 +556,7 @@ xen_global_systime_ns(void)
} while (atomic_cas_64(&xen_global_systime_ns_stamp, global, result)
!= global);
KASSERT(ci == curcpu());
- curlwp_bindx(bound);
+ splx(s);
return result;
}
Added files:
Index: src/sys/arch/x86/x86/x86_softintr.c
diff -u /dev/null src/sys/arch/x86/x86/x86_softintr.c:1.1.2.1
--- /dev/null Sat Apr 11 18:26:08 2020
+++ src/sys/arch/x86/x86/x86_softintr.c Sat Apr 11 18:26:07 2020
@@ -0,0 +1,289 @@
+/* $NetBSD: x86_softintr.c,v 1.1.2.1 2020/04/11 18:26:07 bouyer Exp $ */
+
+/*
+ * Copyright (c) 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Andrew Doran, and by Jason R. Thorpe.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright 2002 (c) Wasabi Systems, Inc.
+ * All rights reserved.
+ *
+ * Written by Frank van der Linden for Wasabi Systems, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed for the NetBSD Project by
+ * Wasabi Systems, Inc.
+ * 4. The name of Wasabi Systems, Inc. may not be used to endorse
+ * or promote products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*-
+ * Copyright (c) 1991 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)isa.c 7.2 (Berkeley) 5/13/91
+ */
+
+/*-
+ * Copyright (c) 1993, 1994 Charles Hannum.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by the University of
+ * California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * @(#)isa.c 7.2 (Berkeley) 5/13/91
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: x86_softintr.c,v 1.1.2.1 2020/04/11 18:26:07 bouyer Exp $");
+
+#include <sys/kmem.h>
+#include <sys/proc.h>
+#include <sys/intr.h>
+
+struct pic softintr_pic = {
+ .pic_name = "softintr_fakepic",
+ .pic_type = PIC_SOFT,
+ .pic_vecbase = 0,
+ .pic_apicid = 0,
+ .pic_lock = __SIMPLELOCK_UNLOCKED,
+};
+
+/*
+ * Recalculate the interrupt masks from scratch.
+ * During early boot, anything goes and we are always called on the BP.
+ * When the system is up and running:
+ *
+ * => called with ci == curcpu()
+ * => cpu_lock held by the initiator
+ * => interrupts disabled on-chip (PSL_I)
+ *
+ * Do not call printf(), kmem_free() or other "heavyweight" routines
+ * from here. This routine must be quick and must not block.
+ */
+void
+x86_intr_calculatemasks(struct cpu_info *ci)
+{
+ int irq, level, unusedirqs, intrlevel[MAX_INTR_SOURCES];
+ struct intrhand *q;
+
+ /* First, figure out which levels each IRQ uses. */
+ unusedirqs = 0xffffffff;
+ for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
+ int levels = 0;
+
+ if (ci->ci_isources[irq] == NULL) {
+ intrlevel[irq] = 0;
+ continue;
+ }
+ for (q = ci->ci_isources[irq]->is_handlers; q; q = q->ih_next)
+ levels |= 1U << q->ih_level;
+ intrlevel[irq] = levels;
+ if (levels)
+ unusedirqs &= ~(1U << irq);
+ }
+
+ /* Then figure out which IRQs use each level. */
+ for (level = 0; level < NIPL; level++) {
+ int irqs = 0;
+ for (irq = 0; irq < MAX_INTR_SOURCES; irq++)
+ if (intrlevel[irq] & (1U << level))
+ irqs |= 1U << irq;
+ ci->ci_imask[level] = irqs | unusedirqs;
+ }
+
+ for (level = 0; level<(NIPL-1); level++)
+ ci->ci_imask[level+1] |= ci->ci_imask[level];
+
+ for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
+ int maxlevel = IPL_NONE;
+ int minlevel = IPL_HIGH;
+
+ if (ci->ci_isources[irq] == NULL)
+ continue;
+ for (q = ci->ci_isources[irq]->is_handlers; q;
+ q = q->ih_next) {
+ if (q->ih_level < minlevel)
+ minlevel = q->ih_level;
+ if (q->ih_level > maxlevel)
+ maxlevel = q->ih_level;
+ }
+ ci->ci_isources[irq]->is_maxlevel = maxlevel;
+ ci->ci_isources[irq]->is_minlevel = minlevel;
+ }
+
+ for (level = 0; level < NIPL; level++)
+ ci->ci_iunmask[level] = ~ci->ci_imask[level];
+}
+
+
+
+#if defined(__HAVE_PREEMPTION)
+struct intrhand fake_preempt_intrhand;
+
+void
+x86_init_preempt(struct cpu_info *ci)
+{
+ struct intrsource *isp;
+ isp = kmem_zalloc(sizeof(*isp), KM_SLEEP);
+ isp->is_recurse = Xrecurse_preempt;
+ isp->is_resume = Xresume_preempt;
+ fake_preempt_intrhand.ih_level = IPL_PREEMPT;
+ isp->is_handlers = &fake_preempt_intrhand;
+ isp->is_pic = &softintr_pic;
+ ci->ci_isources[SIR_PREEMPT] = isp;
+}
+#endif
+
+#if defined(__HAVE_FAST_SOFTINTS)
+struct intrhand fake_softclock_intrhand;
+struct intrhand fake_softnet_intrhand;
+struct intrhand fake_softserial_intrhand;
+struct intrhand fake_softbio_intrhand;
+
+void
+softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
+{
+ struct intrsource *isp;
+ struct cpu_info *ci;
+ u_int sir;
+
+ ci = l->l_cpu;
+
+ isp = kmem_zalloc(sizeof(*isp), KM_SLEEP);
+ isp->is_recurse = Xsoftintr;
+ isp->is_resume = Xsoftintr;
+ isp->is_pic = &softintr_pic;
+
+ switch (level) {
+ case SOFTINT_BIO:
+ sir = SIR_BIO;
+ fake_softbio_intrhand.ih_level = IPL_SOFTBIO;
+ isp->is_handlers = &fake_softbio_intrhand;
+ break;
+ case SOFTINT_NET:
+ sir = SIR_NET;
+ fake_softnet_intrhand.ih_level = IPL_SOFTNET;
+ isp->is_handlers = &fake_softnet_intrhand;
+ break;
+ case SOFTINT_SERIAL:
+ sir = SIR_SERIAL;
+ fake_softserial_intrhand.ih_level = IPL_SOFTSERIAL;
+ isp->is_handlers = &fake_softserial_intrhand;
+ break;
+ case SOFTINT_CLOCK:
+ sir = SIR_CLOCK;
+ fake_softclock_intrhand.ih_level = IPL_SOFTCLOCK;
+ isp->is_handlers = &fake_softclock_intrhand;
+ break;
+ default:
+ panic("softint_init_md");
+ }
+
+ KASSERT(ci->ci_isources[sir] == NULL);
+
+ *machdep = (1 << sir);
+ ci->ci_isources[sir] = isp;
+ ci->ci_isources[sir]->is_lwp = l;
+
+ x86_intr_calculatemasks(ci);
+}
+#endif /* __HAVE_FAST_SOFTINTS */
