Module Name: src
Committed By: yamaguchi
Date: Tue Jul 14 00:45:53 UTC 2020
Modified Files:
src/sys/arch/amd64/amd64: db_interface.c machdep.c
src/sys/arch/amd64/conf: ALL
src/sys/arch/amd64/include: segments.h
src/sys/arch/i386/conf: ALL
src/sys/arch/i386/i386: db_interface.c machdep.c trap.c
src/sys/arch/i386/include: segments.h
src/sys/arch/x86/conf: files.x86
src/sys/arch/x86/include: cpu.h cpuvar.h pmap.h
src/sys/arch/x86/x86: cpu.c hyperv.c idt.c intr.c lapic.c pmap.c svs.c
src/sys/arch/xen/x86: cpu.c
src/sys/arch/xen/xen: hypervisor.c
src/sys/dev/hyperv: vmbusvar.h
src/sys/dev/nvmm/x86: nvmm_x86_vmx.c
Log Message:
Introduce per-cpu IDTs
This is realized by following modifications:
- Add IDT pages and its allocation maps for each cpu in "struct cpu_info"
- Load per-cpu IDTs at cpu_init_idt(struct cpu_info*)
- Copy the IDT entries for cpu0 to other CPUs at attach
- These are, for example, exceptions, db, system calls, etc.
And, added a kernel option named PCPU_IDT to enable the feature.
To generate a diff of this commit:
cvs rdiff -u -r1.37 -r1.38 src/sys/arch/amd64/amd64/db_interface.c
cvs rdiff -u -r1.355 -r1.356 src/sys/arch/amd64/amd64/machdep.c
cvs rdiff -u -r1.155 -r1.156 src/sys/arch/amd64/conf/ALL
cvs rdiff -u -r1.36 -r1.37 src/sys/arch/amd64/include/segments.h
cvs rdiff -u -r1.493 -r1.494 src/sys/arch/i386/conf/ALL
cvs rdiff -u -r1.84 -r1.85 src/sys/arch/i386/i386/db_interface.c
cvs rdiff -u -r1.830 -r1.831 src/sys/arch/i386/i386/machdep.c
cvs rdiff -u -r1.303 -r1.304 src/sys/arch/i386/i386/trap.c
cvs rdiff -u -r1.68 -r1.69 src/sys/arch/i386/include/segments.h
cvs rdiff -u -r1.116 -r1.117 src/sys/arch/x86/conf/files.x86
cvs rdiff -u -r1.126 -r1.127 src/sys/arch/x86/include/cpu.h
cvs rdiff -u -r1.52 -r1.53 src/sys/arch/x86/include/cpuvar.h
cvs rdiff -u -r1.123 -r1.124 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.194 -r1.195 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.10 -r1.11 src/sys/arch/x86/x86/hyperv.c
cvs rdiff -u -r1.12 -r1.13 src/sys/arch/x86/x86/idt.c
cvs rdiff -u -r1.151 -r1.152 src/sys/arch/x86/x86/intr.c
cvs rdiff -u -r1.83 -r1.84 src/sys/arch/x86/x86/lapic.c
cvs rdiff -u -r1.399 -r1.400 src/sys/arch/x86/x86/pmap.c
cvs rdiff -u -r1.37 -r1.38 src/sys/arch/x86/x86/svs.c
cvs rdiff -u -r1.138 -r1.139 src/sys/arch/xen/x86/cpu.c
cvs rdiff -u -r1.86 -r1.87 src/sys/arch/xen/xen/hypervisor.c
cvs rdiff -u -r1.5 -r1.6 src/sys/dev/hyperv/vmbusvar.h
cvs rdiff -u -r1.61 -r1.62 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/db_interface.c
diff -u src/sys/arch/amd64/amd64/db_interface.c:1.37 src/sys/arch/amd64/amd64/db_interface.c:1.38
--- src/sys/arch/amd64/amd64/db_interface.c:1.37 Sat Feb 29 15:00:28 2020
+++ src/sys/arch/amd64/amd64/db_interface.c Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: db_interface.c,v 1.37 2020/02/29 15:00:28 christos Exp $ */
+/* $NetBSD: db_interface.c,v 1.38 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Mach Operating System
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.37 2020/02/29 15:00:28 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.38 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -111,12 +111,14 @@ db_machine_init(void)
#ifdef MULTIPROCESSOR
#ifndef XENPV
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
vector *handler = &Xintr_ddbipi;
+ idt_descriptor_t *idt = iv->iv_idt;
#if NLAPIC > 0
if (lapic_is_x2apic())
handler = &Xintr_x2apic_ddbipi;
#endif
- ddb_vec = idt_vec_alloc(0xf0, 0xff);
+ ddb_vec = idt_vec_alloc(iv, 0xf0, 0xff);
set_idtgate(&idt[ddb_vec], handler, 1, SDT_SYS386IGT, SEL_KPL,
GSEL(GCODE_SEL, SEL_KPL));
#else
Index: src/sys/arch/amd64/amd64/machdep.c
diff -u src/sys/arch/amd64/amd64/machdep.c:1.355 src/sys/arch/amd64/amd64/machdep.c:1.356
--- src/sys/arch/amd64/amd64/machdep.c:1.355 Sun May 10 06:30:57 2020
+++ src/sys/arch/amd64/amd64/machdep.c Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.355 2020/05/10 06:30:57 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.356 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.355 2020/05/10 06:30:57 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.356 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_modular.h"
#include "opt_user_ldt.h"
@@ -1392,11 +1392,15 @@ char *ldtstore;
char *gdtstore;
void
-setgate(struct gate_descriptor *gd, void *func, int ist, int type, int dpl, int sel)
+setgate(struct gate_descriptor *gd, void *func,
+ int ist, int type, int dpl, int sel)
{
+ vaddr_t vaddr;
+
+ vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
kpreempt_disable();
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+ pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
gd->gd_looffset = (uint64_t)func & 0xffff;
gd->gd_selector = sel;
@@ -1410,20 +1414,23 @@ setgate(struct gate_descriptor *gd, void
gd->gd_xx2 = 0;
gd->gd_xx3 = 0;
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
+ pmap_changeprot_local(vaddr, VM_PROT_READ);
kpreempt_enable();
}
void
unsetgate(struct gate_descriptor *gd)
{
+ vaddr_t vaddr;
+
+ vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
kpreempt_disable();
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+ pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
memset(gd, 0, sizeof (*gd));
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
+ pmap_changeprot_local(vaddr, VM_PROT_READ);
kpreempt_enable();
}
@@ -1470,10 +1477,12 @@ set_sys_segment(struct sys_segment_descr
}
void
-cpu_init_idt(void)
+cpu_init_idt(struct cpu_info *ci)
{
struct region_descriptor region;
+ idt_descriptor_t *idt;
+ idt = ci->ci_idtvec.iv_idt;
setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1);
lidt(®ion);
}
@@ -1677,6 +1686,8 @@ init_x86_64(paddr_t first_avail)
extern void consinit(void);
struct region_descriptor region;
struct mem_segment_descriptor *ldt_segp;
+ struct idt_vec *iv;
+ idt_descriptor_t *idt;
int x;
struct pcb *pcb;
extern vaddr_t lwp0uarea;
@@ -1807,7 +1818,9 @@ init_x86_64(paddr_t first_avail)
pmap_update(pmap_kernel());
- idt = (idt_descriptor_t *)idt_vaddr;
+ iv = &(cpu_info_primary.ci_idtvec);
+ idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary));
+ idt = iv->iv_idt;
gdtstore = (char *)gdt_vaddr;
ldtstore = (char *)ldt_vaddr;
@@ -1872,7 +1885,7 @@ init_x86_64(paddr_t first_avail)
sel = SEL_KPL;
ist = 0;
- idt_vec_reserve(x);
+ idt_vec_reserve(iv, x);
switch (x) {
case 1: /* DB */
@@ -1902,7 +1915,7 @@ init_x86_64(paddr_t first_avail)
}
/* new-style interrupt gate for syscalls */
- idt_vec_reserve(128);
+ idt_vec_reserve(iv, 128);
set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL,
GSEL(GCODE_SEL, SEL_KPL));
@@ -1920,7 +1933,7 @@ init_x86_64(paddr_t first_avail)
panic("HYPERVISOR_set_callbacks() failed");
#endif /* XENPV */
- cpu_init_idt();
+ cpu_init_idt(&cpu_info_primary);
#ifdef XENPV
xen_init_ksyms();
@@ -1961,6 +1974,14 @@ init_x86_64(paddr_t first_avail)
void
cpu_reset(void)
{
+#ifndef XENPV
+ idt_descriptor_t *idt;
+ vaddr_t vaddr;
+
+ idt = cpu_info_primary.ci_idtvec.iv_idt;
+ vaddr = (vaddr_t)idt;
+#endif
+
x86_disable_intr();
#ifdef XENPV
@@ -1974,7 +1995,7 @@ cpu_reset(void)
* invalid and causing a fault.
*/
kpreempt_disable();
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+ pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
memset((void *)idt, 0, NIDT * sizeof(idt[0]));
kpreempt_enable();
breakpoint();
@@ -2321,3 +2342,56 @@ mm_md_direct_mapped_phys(paddr_t paddr,
return false;
#endif
}
+
+static void
+idt_vec_copy(struct idt_vec *dst, struct idt_vec *src)
+{
+ idt_descriptor_t *idt_dst;
+
+ idt_dst = dst->iv_idt;
+
+ kpreempt_disable();
+ pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE);
+
+ memcpy(idt_dst, src->iv_idt, PAGE_SIZE);
+ memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap));
+
+ pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ);
+ kpreempt_enable();
+}
+
+void
+idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid)
+{
+ vaddr_t va;
+
+ if (cid != cpu_index(&cpu_info_primary) &&
+ idt_vec_is_pcpu()) {
+#ifdef __HAVE_PCPU_AREA
+ va = (vaddr_t)&pcpuarea->ent[cid].idt;
+#else
+ struct vm_page *pg;
+
+ va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
+ UVM_KMF_VAONLY);
+ pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
+ if (pg == NULL) {
+ panic("failed to allocate a page for IDT");
+ }
+ pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
+ VM_PROT_READ|VM_PROT_WRITE, 0);
+ pmap_update(pmap_kernel());
+#endif
+
+ memset((void *)va, 0, PAGE_SIZE);
+#ifndef XENPV
+ pmap_changeprot_local(va, VM_PROT_READ);
+#endif
+ pmap_update(pmap_kernel());
+
+ iv->iv_idt = (void *)va;
+ idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec));
+ } else {
+ iv->iv_idt = (void *)idt_vaddr;
+ }
+}
Index: src/sys/arch/amd64/conf/ALL
diff -u src/sys/arch/amd64/conf/ALL:1.155 src/sys/arch/amd64/conf/ALL:1.156
--- src/sys/arch/amd64/conf/ALL:1.155 Mon Jun 29 23:58:44 2020
+++ src/sys/arch/amd64/conf/ALL Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-# $NetBSD: ALL,v 1.155 2020/06/29 23:58:44 riastradh Exp $
+# $NetBSD: ALL,v 1.156 2020/07/14 00:45:52 yamaguchi Exp $
# From NetBSD: GENERIC,v 1.787 2006/10/01 18:37:54 bouyer Exp
#
# ALL machine description file
@@ -17,7 +17,7 @@ include "arch/amd64/conf/std.amd64"
options INCLUDE_CONFIG_FILE # embed config file in kernel binary
-#ident "ALL-$Revision: 1.155 $"
+#ident "ALL-$Revision: 1.156 $"
maxusers 64 # estimated number of users
@@ -29,6 +29,7 @@ options X86EMU # 386 Real Mode emulato
#options PAE # PAE mode (36 bits physical addressing)
makeoptions SPECTRE_V2_GCC_MITIGATION=1 # GCC Spectre variant 2
# migitation
+options PCPU_IDT # Per CPU IDT
# CPU features
acpicpu* at cpu? # ACPI CPU (including frequency scaling)
Index: src/sys/arch/amd64/include/segments.h
diff -u src/sys/arch/amd64/include/segments.h:1.36 src/sys/arch/amd64/include/segments.h:1.37
--- src/sys/arch/amd64/include/segments.h:1.36 Mon Feb 11 14:59:32 2019
+++ src/sys/arch/amd64/include/segments.h Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: segments.h,v 1.36 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: segments.h,v 1.37 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Copyright (c) 1990 The Regents of the University of California.
@@ -238,7 +238,6 @@ typedef struct trap_info idt_descriptor_
#else
typedef struct gate_descriptor idt_descriptor_t;
#endif /* XENPV */
-extern idt_descriptor_t *idt;
extern char *gdtstore;
extern char *ldtstore;
@@ -251,14 +250,16 @@ void set_sys_segment(struct sys_segment_
int, int, int);
void set_mem_segment(struct mem_segment_descriptor *, void *, size_t,
int, int, int, int, int);
-void cpu_init_idt(void);
void update_descriptor(void *, void *);
-
-void idt_vec_reserve(int);
-int idt_vec_alloc(int, int);
-void idt_vec_set(int, void (*)(void));
-void idt_vec_free(int);
+struct idt_vec;
+void idt_vec_reserve(struct idt_vec *, int);
+int idt_vec_alloc(struct idt_vec *, int, int);
+void idt_vec_set(struct idt_vec *, int, void (*)(void));
+void idt_vec_free(struct idt_vec *, int);
+void idt_vec_init_cpu_md(struct idt_vec *, cpuid_t);
+bool idt_vec_is_pcpu(void);
+struct idt_vec * idt_vec_ref(struct idt_vec *);
struct lwp;
Index: src/sys/arch/i386/conf/ALL
diff -u src/sys/arch/i386/conf/ALL:1.493 src/sys/arch/i386/conf/ALL:1.494
--- src/sys/arch/i386/conf/ALL:1.493 Mon May 25 07:20:15 2020
+++ src/sys/arch/i386/conf/ALL Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-# $NetBSD: ALL,v 1.493 2020/05/25 07:20:15 yamaguchi Exp $
+# $NetBSD: ALL,v 1.494 2020/07/14 00:45:52 yamaguchi Exp $
# From NetBSD: GENERIC,v 1.787 2006/10/01 18:37:54 bouyer Exp
#
# ALL machine description file
@@ -17,7 +17,7 @@ include "arch/i386/conf/std.i386"
options INCLUDE_CONFIG_FILE # embed config file in kernel binary
-#ident "ALL-$Revision: 1.493 $"
+#ident "ALL-$Revision: 1.494 $"
maxusers 64 # estimated number of users
@@ -29,6 +29,7 @@ options X86EMU # 386 Real Mode emulato
options PAE # PAE mode (36 bits physical addressing)
makeoptions SPECTRE_V2_GCC_MITIGATION=1 # GCC Spectre variant 2
# migitation
+options PCPU_IDT # Per CPU IDT
# CPU features
acpicpu* at cpu? # ACPI CPU (including frequency scaling)
Index: src/sys/arch/i386/i386/db_interface.c
diff -u src/sys/arch/i386/i386/db_interface.c:1.84 src/sys/arch/i386/i386/db_interface.c:1.85
--- src/sys/arch/i386/i386/db_interface.c:1.84 Thu Feb 14 07:12:40 2019
+++ src/sys/arch/i386/i386/db_interface.c Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: db_interface.c,v 1.84 2019/02/14 07:12:40 cherry Exp $ */
+/* $NetBSD: db_interface.c,v 1.85 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Mach Operating System
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.84 2019/02/14 07:12:40 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.85 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -114,12 +114,15 @@ db_machine_init(void)
#ifdef MULTIPROCESSOR
#ifndef XENPV
vector *handler = &Xintr_ddbipi;
+ struct idt_vec *iv;
+
+ iv = &(cpu_info_primary.ci_idtvec);
#if NLAPIC > 0
if (lapic_is_x2apic())
handler = &Xintr_x2apic_ddbipi;
#endif
- ddb_vec = idt_vec_alloc(0xf0, 0xff);
- idt_vec_set(ddb_vec, handler);
+ ddb_vec = idt_vec_alloc(iv, 0xf0, 0xff);
+ idt_vec_set(iv, ddb_vec, handler);
#else
/* Initialised as part of xen_ipi_init() */
#endif /* XENPV */
Index: src/sys/arch/i386/i386/machdep.c
diff -u src/sys/arch/i386/i386/machdep.c:1.830 src/sys/arch/i386/i386/machdep.c:1.831
--- src/sys/arch/i386/i386/machdep.c:1.830 Fri May 8 00:52:29 2020
+++ src/sys/arch/i386/i386/machdep.c Tue Jul 14 00:45:52 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.830 2020/05/08 00:52:29 riastradh Exp $ */
+/* $NetBSD: machdep.c,v 1.831 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2004, 2006, 2008, 2009, 2017
@@ -67,7 +67,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.830 2020/05/08 00:52:29 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.831 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_beep.h"
#include "opt_compat_freebsd.h"
@@ -554,6 +554,7 @@ cpu_set_tss_gates(struct cpu_info *ci)
{
struct segment_descriptor sd;
void *doubleflt_stack;
+ idt_descriptor_t *idt;
doubleflt_stack = (void *)uvm_km_alloc(kernel_map, USPACE, 0,
UVM_KMF_WIRED);
@@ -563,6 +564,7 @@ cpu_set_tss_gates(struct cpu_info *ci)
SDT_SYS386TSS, SEL_KPL, 0, 0);
ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
+ idt = cpu_info_primary.ci_idtvec.iv_idt;
set_idtgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
GSEL(GTRAPTSS_SEL, SEL_KPL));
@@ -906,6 +908,7 @@ setgate(struct gate_descriptor *gd, void
void
unsetgate(struct gate_descriptor *gd)
{
+
gd->gd_p = 0;
gd->gd_hioffset = 0;
gd->gd_looffset = 0;
@@ -949,10 +952,15 @@ extern union descriptor tmpgdt[];
#endif
void
-cpu_init_idt(void)
+cpu_init_idt(struct cpu_info *ci)
{
struct region_descriptor region;
- setregion(®ion, pentium_idt, NIDT * sizeof(idt[0]) - 1);
+ struct idt_vec *iv;
+ idt_descriptor_t *idt;
+
+ iv = &ci->ci_idtvec;
+ idt = iv->iv_idt_pentium;
+ setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1);
lidt(®ion);
}
@@ -1134,6 +1142,8 @@ init386(paddr_t first_avail)
#endif
#endif /* !XENPV */
struct pcb *pcb;
+ struct idt_vec *iv;
+ idt_descriptor_t *idt;
KASSERT(first_avail % PAGE_SIZE == 0);
@@ -1305,8 +1315,9 @@ init386(paddr_t first_avail)
pmap_kenter_pa(pentium_idt_vaddr, idt_paddr, VM_PROT_READ, 0);
pmap_update(pmap_kernel());
- pentium_idt = (union descriptor *)pentium_idt_vaddr;
- idt = (idt_descriptor_t *)idt_vaddr;
+ iv = &(cpu_info_primary.ci_idtvec);
+ idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary));
+ idt = (idt_descriptor_t *)iv->iv_idt;
#ifndef XENPV
tgdt = gdtstore;
@@ -1340,7 +1351,7 @@ init386(paddr_t first_avail)
sel = SEL_KPL;
#endif /* XENPV */
- idt_vec_reserve(x);
+ idt_vec_reserve(iv, x);
switch (x) {
#ifdef XENPV
@@ -1361,7 +1372,7 @@ init386(paddr_t first_avail)
}
/* new-style interrupt gate for syscalls */
- idt_vec_reserve(128);
+ idt_vec_reserve(iv, 128);
set_idtgate(&idt[128], &IDTVEC(syscall), 0, SDT_SYS386IGT, SEL_UPL,
GSEL(GCODE_SEL, SEL_KPL));
@@ -1371,7 +1382,7 @@ init386(paddr_t first_avail)
#endif
lldt(GSEL(GLDT_SEL, SEL_KPL));
- cpu_init_idt();
+ cpu_init_idt(&cpu_info_primary);
#ifdef XENPV
xen_init_ksyms();
@@ -1441,7 +1452,9 @@ cpu_reset(void)
for (;;);
#else /* XENPV */
struct region_descriptor region;
+ idt_descriptor_t *idt;
+ idt = (idt_descriptor_t *)cpu_info_primary.ci_idtvec.iv_idt;
x86_disable_intr();
/*
@@ -1698,3 +1711,50 @@ cpu_alloc_l3_page(struct cpu_info *ci)
pmap_update(pmap_kernel());
}
#endif /* PAE */
+
+static void
+idt_vec_copy(struct idt_vec *dst, struct idt_vec *src)
+{
+ idt_descriptor_t *idt_dst;
+
+ idt_dst = dst->iv_idt;
+ memcpy(idt_dst, src->iv_idt, PAGE_SIZE);
+ memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap));
+}
+
+void
+idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid)
+{
+ vaddr_t va_idt, va_pentium_idt;
+ struct vm_page *pg;
+
+ if (idt_vec_is_pcpu() &&
+ cid != cpu_index(&cpu_info_primary)) {
+ va_idt = uvm_km_alloc(kernel_map, PAGE_SIZE,
+ 0, UVM_KMF_VAONLY);
+ pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
+ if (pg == NULL) {
+ panic("failed to allocate pcpu idt PA");
+ }
+ pmap_kenter_pa(va_idt, VM_PAGE_TO_PHYS(pg),
+ VM_PROT_READ|VM_PROT_WRITE, 0);
+ pmap_update(pmap_kernel());
+
+ memset((void *)va_idt, 0, PAGE_SIZE);
+
+ /* pentium f00f bug stuff */
+ va_pentium_idt = uvm_km_alloc(kernel_map, PAGE_SIZE,
+ 0, UVM_KMF_VAONLY);
+ pmap_kenter_pa(va_pentium_idt, VM_PAGE_TO_PHYS(pg),
+ VM_PROT_READ, 0);
+ pmap_update(pmap_kernel());
+
+ iv->iv_idt = (void *)va_idt;
+ iv->iv_idt_pentium = (void *)va_pentium_idt;
+
+ idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec));
+ } else {
+ iv->iv_idt = (void *)idt_vaddr;
+ iv->iv_idt_pentium = (void *)pentium_idt_vaddr;
+ }
+}
Index: src/sys/arch/i386/i386/trap.c
diff -u src/sys/arch/i386/i386/trap.c:1.303 src/sys/arch/i386/i386/trap.c:1.304
--- src/sys/arch/i386/i386/trap.c:1.303 Thu Nov 21 19:24:00 2019
+++ src/sys/arch/i386/i386/trap.c Tue Jul 14 00:45:52 2020
@@ -1,5 +1,5 @@
-/* $NetBSD: trap.c,v 1.303 2019/11/21 19:24:00 ad Exp $ */
+/* $NetBSD: trap.c,v 1.304 2020/07/14 00:45:52 yamaguchi Exp $ */
/*-
* Copyright (c) 1998, 2000, 2005, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -69,7 +69,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.303 2019/11/21 19:24:00 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.304 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_ddb.h"
#include "opt_kgdb.h"
@@ -245,10 +245,16 @@ int
ss_shadow(struct trapframe *tf)
{
struct gate_descriptor *gd;
+ struct cpu_info *ci;
+ struct idt_vec *iv;
+ idt_descriptor_t *idt;
uintptr_t eip, func;
size_t i;
eip = tf->tf_eip;
+ ci = curcpu();
+ iv = idt_vec_ref(&ci->ci_idtvec);
+ idt = iv->iv_idt;
for (i = 0; i < 256; i++) {
gd = &idt[i];
Index: src/sys/arch/i386/include/segments.h
diff -u src/sys/arch/i386/include/segments.h:1.68 src/sys/arch/i386/include/segments.h:1.69
--- src/sys/arch/i386/include/segments.h:1.68 Mon Feb 11 14:59:32 2019
+++ src/sys/arch/i386/include/segments.h Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: segments.h,v 1.68 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: segments.h,v 1.69 2020/07/14 00:45:53 yamaguchi Exp $ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
@@ -196,7 +196,6 @@ typedef struct trap_info idt_descriptor_
#else
typedef struct gate_descriptor idt_descriptor_t;
#endif /* XENPV */
-extern idt_descriptor_t *idt;
extern union descriptor *gdtstore, *ldtstore;
void setgate(struct gate_descriptor *, void *, int, int, int, int);
@@ -206,15 +205,16 @@ void setregion(struct region_descriptor
void setsegment(struct segment_descriptor *, const void *, size_t, int, int,
int, int);
void unsetgate(struct gate_descriptor *);
-
-void cpu_init_idt(void);
void update_descriptor(union descriptor *, union descriptor *);
-
-void idt_vec_reserve(int);
-int idt_vec_alloc(int, int);
-void idt_vec_set(int, void (*)(void));
-void idt_vec_free(int);
+struct idt_vec;
+void idt_vec_reserve(struct idt_vec *, int);
+int idt_vec_alloc(struct idt_vec *, int, int);
+void idt_vec_set(struct idt_vec *, int, void (*)(void));
+void idt_vec_free(struct idt_vec *, int);
+void idt_vec_init_cpu_md(struct idt_vec *, cpuid_t);
+bool idt_vec_is_pcpu(void);
+struct idt_vec* idt_vec_ref(struct idt_vec *);
#endif /* _KERNEL */
Index: src/sys/arch/x86/conf/files.x86
diff -u src/sys/arch/x86/conf/files.x86:1.116 src/sys/arch/x86/conf/files.x86:1.117
--- src/sys/arch/x86/conf/files.x86:1.116 Mon Jun 29 23:51:35 2020
+++ src/sys/arch/x86/conf/files.x86 Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.x86,v 1.116 2020/06/29 23:51:35 riastradh Exp $
+# $NetBSD: files.x86,v 1.117 2020/07/14 00:45:53 yamaguchi Exp $
# options for MP configuration through the MP spec
defflag opt_mpbios.h MPBIOS MPDEBUG MPBIOS_SCANPCI
@@ -22,6 +22,8 @@ defflag opt_kaslr.h NO_X86_ASLR
defflag SVS
+defflag PCPU_IDT
+
define cpubus { [apid = -1] }
define cpufeaturebus {}
define ioapicbus { [apid = -1] }
Index: src/sys/arch/x86/include/cpu.h
diff -u src/sys/arch/x86/include/cpu.h:1.126 src/sys/arch/x86/include/cpu.h:1.127
--- src/sys/arch/x86/include/cpu.h:1.126 Fri Jun 19 16:20:22 2020
+++ src/sys/arch/x86/include/cpu.h Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.126 2020/06/19 16:20:22 maxv Exp $ */
+/* $NetBSD: cpu.h,v 1.127 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 1990 The Regents of the University of California.
@@ -103,6 +103,12 @@ struct clockframe {
struct intrframe cf_if;
};
+struct idt_vec {
+ void *iv_idt;
+ void *iv_idt_pentium;
+ char iv_allocmap[NIDT];
+};
+
/*
* a bunch of this belongs in cpuvar.h; move it later..
*/
@@ -126,6 +132,7 @@ struct cpu_info {
uint64_t ci_scratch;
uintptr_t ci_pmap_data[128 / sizeof(uintptr_t)];
struct kcpuset *ci_tlb_cpuset;
+ struct idt_vec ci_idtvec;
int ci_kfpu_spl;
Index: src/sys/arch/x86/include/cpuvar.h
diff -u src/sys/arch/x86/include/cpuvar.h:1.52 src/sys/arch/x86/include/cpuvar.h:1.53
--- src/sys/arch/x86/include/cpuvar.h:1.52 Sat Apr 25 15:26:18 2020
+++ src/sys/arch/x86/include/cpuvar.h Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpuvar.h,v 1.52 2020/04/25 15:26:18 bouyer Exp $ */
+/* $NetBSD: cpuvar.h,v 1.53 2020/07/14 00:45:53 yamaguchi Exp $ */
/*-
* Copyright (c) 2000, 2007 The NetBSD Foundation, Inc.
@@ -112,6 +112,7 @@ void identifycpu_cpuids(struct cpu_info
void cpu_init(struct cpu_info *);
void cpu_init_tss(struct cpu_info *);
void cpu_init_first(void);
+void cpu_init_idt(struct cpu_info *);
void x86_cpu_idle_init(void);
void x86_cpu_idle_halt(void);
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.123 src/sys/arch/x86/include/pmap.h:1.124
--- src/sys/arch/x86/include/pmap.h:1.123 Wed Jun 24 18:09:37 2020
+++ src/sys/arch/x86/include/pmap.h Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.123 2020/06/24 18:09:37 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.124 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -198,6 +198,7 @@ extern struct slotspace slotspace;
struct pcpu_entry {
uint8_t gdt[MAXGDTSIZ];
uint8_t ldt[MAX_USERLDT_SIZE];
+ uint8_t idt[PAGE_SIZE];
uint8_t tss[PAGE_SIZE];
uint8_t ist0[PAGE_SIZE];
uint8_t ist1[PAGE_SIZE];
@@ -210,7 +211,6 @@ struct pcpu_area {
#ifdef SVS
uint8_t utls[PAGE_SIZE];
#endif
- uint8_t idt[PAGE_SIZE];
uint8_t ldt[PAGE_SIZE];
struct pcpu_entry ent[MAXCPUS];
} __packed;
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.194 src/sys/arch/x86/x86/cpu.c:1.195
--- src/sys/arch/x86/x86/cpu.c:1.194 Mon Jun 15 09:09:24 2020
+++ src/sys/arch/x86/x86/cpu.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.194 2020/06/15 09:09:24 msaitoh Exp $ */
+/* $NetBSD: cpu.c,v 1.195 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 2000-2020 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.194 2020/06/15 09:09:24 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.195 2020/07/14 00:45:53 yamaguchi Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -493,6 +493,7 @@ cpu_attach(device_t parent, device_t sel
* report on an AP
*/
cpu_intr_init(ci);
+ idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci));
gdt_alloc_cpu(ci);
#ifdef i386
cpu_set_tss_gates(ci);
@@ -978,7 +979,7 @@ cpu_hatch(void *v)
pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
lcr0(pcb->pcb_cr0);
- cpu_init_idt();
+ cpu_init_idt(ci);
gdt_init_cpu(ci);
#if NLAPIC > 0
lapic_enable();
Index: src/sys/arch/x86/x86/hyperv.c
diff -u src/sys/arch/x86/x86/hyperv.c:1.10 src/sys/arch/x86/x86/hyperv.c:1.11
--- src/sys/arch/x86/x86/hyperv.c:1.10 Mon Jun 15 09:09:24 2020
+++ src/sys/arch/x86/x86/hyperv.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: hyperv.c,v 1.10 2020/06/15 09:09:24 msaitoh Exp $ */
+/* $NetBSD: hyperv.c,v 1.11 2020/07/14 00:45:53 yamaguchi Exp $ */
/*-
* Copyright (c) 2009-2012,2016-2017 Microsoft Corp.
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
#ifdef __KERNEL_RCSID
-__KERNEL_RCSID(0, "$NetBSD: hyperv.c,v 1.10 2020/06/15 09:09:24 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hyperv.c,v 1.11 2020/07/14 00:45:53 yamaguchi Exp $");
#endif
#ifdef __FBSDID
__FBSDID("$FreeBSD: head/sys/dev/hyperv/vmbus/hyperv.c 331757 2018-03-30 02:25:12Z emaste $");
@@ -95,6 +95,10 @@ struct hyperv_hypercall_ctx {
paddr_t hc_paddr;
};
+struct hyperv_percpu_data {
+ int pd_idtvec;
+};
+
static struct hyperv_hypercall_ctx hyperv_hypercall_ctx;
static char hyperv_hypercall_page[PAGE_SIZE]
@@ -115,8 +119,6 @@ static char hyperv_features_str[256];
static char hyperv_pm_features_str[256];
static char hyperv_features3_str[256];
-static int hyperv_idtvec;
-
uint32_t hyperv_vcpuid[MAXCPUS];
static struct timecounter hyperv_timecounter = {
@@ -756,37 +758,91 @@ void
vmbus_init_interrupts_md(struct vmbus_softc *sc)
{
extern void Xintr_hyperv_hypercall(void);
+ struct vmbus_percpu_data *pd;
+ struct hyperv_percpu_data *hv_pd;
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
+ cpuid_t cid;
+ if (idt_vec_is_pcpu())
+ return;
/*
* All Hyper-V ISR required resources are setup, now let's find a
* free IDT vector for Hyper-V ISR and set it up.
*/
+ iv = &(cpu_info_primary.ci_idtvec);
+ cid = cpu_index(&cpu_info_primary);
+ pd = &sc->sc_percpu[cid];
+
+ hv_pd = kmem_zalloc(sizeof(*hv_pd), KM_SLEEP);
mutex_enter(&cpu_lock);
- hyperv_idtvec = idt_vec_alloc(APIC_LEVEL(NIPL), IDT_INTR_HIGH);
+ hv_pd->pd_idtvec = idt_vec_alloc(iv,
+ APIC_LEVEL(NIPL), IDT_INTR_HIGH);
mutex_exit(&cpu_lock);
- KASSERT(hyperv_idtvec > 0);
- idt_vec_set(hyperv_idtvec, Xintr_hyperv_hypercall);
+ KASSERT(hv_pd->pd_idtvec > 0);
+ idt_vec_set(iv, hv_pd->pd_idtvec, Xintr_hyperv_hypercall);
+ pd->md_cookie = (void *)hv_pd;
}
void
vmbus_deinit_interrupts_md(struct vmbus_softc *sc)
{
+ struct vmbus_percpu_data *pd;
+ struct hyperv_percpu_data *hv_pd;
+ struct idt_vec *iv;
+ cpuid_t cid;
- if (hyperv_idtvec > 0) {
- idt_vec_free(hyperv_idtvec);
- hyperv_idtvec = 0;
- }
+ if (idt_vec_is_pcpu())
+ return;
+
+ iv = &(cpu_info_primary.ci_idtvec);
+ cid = cpu_index(&cpu_info_primary);
+ pd = &sc->sc_percpu[cid];
+ hv_pd = pd->md_cookie;
+
+ if (hv_pd->pd_idtvec > 0)
+ idt_vec_free(iv, hv_pd->pd_idtvec);
+
+ pd->md_cookie = NULL;
+ kmem_free(hv_pd, sizeof(*hv_pd));
}
void
vmbus_init_synic_md(struct vmbus_softc *sc, cpuid_t cpu)
{
- struct vmbus_percpu_data *pd;
+ extern void Xintr_hyperv_hypercall(void);
+ struct vmbus_percpu_data *pd, *pd0;
+ struct hyperv_percpu_data *hv_pd;
+ struct cpu_info *ci;
+ struct idt_vec *iv;
uint64_t val, orig;
uint32_t sint;
+ int hyperv_idtvec;
pd = &sc->sc_percpu[cpu];
+ hv_pd = kmem_alloc(sizeof(*hv_pd), KM_SLEEP);
+ pd->md_cookie = (void *)hv_pd;
+
+ /* Allocate IDT vector for ISR and set it up. */
+ if (idt_vec_is_pcpu()) {
+ ci = curcpu();
+ iv = &ci->ci_idtvec;
+
+ mutex_enter(&cpu_lock);
+ hyperv_idtvec = idt_vec_alloc(iv, APIC_LEVEL(NIPL), IDT_INTR_HIGH);
+ mutex_exit(&cpu_lock);
+ KASSERT(hyperv_idtvec > 0);
+ idt_vec_set(iv, hyperv_idtvec, Xintr_hyperv_hypercall);
+
+ hv_pd = kmem_alloc(sizeof(*hv_pd), KM_SLEEP);
+ hv_pd->pd_idtvec = hyperv_idtvec;
+ pd->md_cookie = hv_pd;
+ } else {
+ pd0 = &sc->sc_percpu[cpu_index(&cpu_info_primary)];
+ hv_pd = pd0->md_cookie;
+ hyperv_idtvec = hv_pd->pd_idtvec;
+ }
+
/*
* Setup the SynIC message.
*/
@@ -832,6 +888,10 @@ vmbus_init_synic_md(struct vmbus_softc *
void
vmbus_deinit_synic_md(struct vmbus_softc *sc, cpuid_t cpu)
{
+ struct vmbus_percpu_data *pd;
+ struct hyperv_percpu_data *hv_pd;
+ struct cpu_info *ci;
+ struct idt_vec *iv;
uint64_t orig;
uint32_t sint;
@@ -866,6 +926,22 @@ vmbus_deinit_synic_md(struct vmbus_softc
*/
orig = rdmsr(MSR_HV_SIEFP);
wrmsr(MSR_HV_SIEFP, (orig & MSR_HV_SIEFP_RSVD_MASK));
+
+ /*
+ * Free IDT vector
+ */
+ if (idt_vec_is_pcpu()) {
+ ci = curcpu();
+ iv = &ci->ci_idtvec;
+ pd = &sc->sc_percpu[cpu_index(ci)];
+ hv_pd = pd->md_cookie;
+
+ if (hv_pd->pd_idtvec > 0)
+ idt_vec_free(iv, hv_pd->pd_idtvec);
+
+ pd->md_cookie = NULL;
+ kmem_free(hv_pd, sizeof(*hv_pd));
+ }
}
static int
Index: src/sys/arch/x86/x86/idt.c
diff -u src/sys/arch/x86/x86/idt.c:1.12 src/sys/arch/x86/x86/idt.c:1.13
--- src/sys/arch/x86/x86/idt.c:1.12 Sat Jul 4 09:03:54 2020
+++ src/sys/arch/x86/x86/idt.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: idt.c,v 1.12 2020/07/04 09:03:54 bouyer Exp $ */
+/* $NetBSD: idt.c,v 1.13 2020/07/14 00:45:53 yamaguchi Exp $ */
/*-
* Copyright (c) 1996, 1997, 1998, 2000, 2009 The NetBSD Foundation, Inc.
@@ -65,7 +65,9 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: idt.c,v 1.12 2020/07/04 09:03:54 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: idt.c,v 1.13 2020/07/14 00:45:53 yamaguchi Exp $");
+
+#include "opt_pcpu_idt.h"
#include <sys/param.h>
#include <sys/systm.h>
@@ -81,9 +83,6 @@ __KERNEL_RCSID(0, "$NetBSD: idt.c,v 1.12
* XEN PV and native have a different idea of what idt entries should
* look like.
*/
-idt_descriptor_t *idt;
-
-static char idt_allocmap[NIDT];
/* Normalise across XEN PV and native */
#if defined(XENPV)
@@ -164,9 +163,10 @@ unset_idtgate(struct gate_descriptor *id
* cpu_lock will be held unless single threaded during early boot.
*/
int
-idt_vec_alloc(int low, int high)
+idt_vec_alloc(struct idt_vec *iv, int low, int high)
{
int vec;
+ char *idt_allocmap = iv->iv_allocmap;
KASSERT(mutex_owned(&cpu_lock) || !mp_online);
@@ -182,23 +182,26 @@ idt_vec_alloc(int low, int high)
}
void
-idt_vec_reserve(int vec)
+idt_vec_reserve(struct idt_vec *iv, int vec)
{
int result;
KASSERT(mutex_owned(&cpu_lock) || !mp_online);
- result = idt_vec_alloc(vec, vec);
+ result = idt_vec_alloc(iv, vec, vec);
if (result != vec) {
panic("%s: failed to reserve vec %d", __func__, vec);
}
}
void
-idt_vec_set(int vec, void (*function)(void))
+idt_vec_set(struct idt_vec *iv, int vec, void (*function)(void))
{
+ idt_descriptor_t *idt;
+ char *idt_allocmap = iv->iv_allocmap;
KASSERT(idt_allocmap[vec] == 1);
+ idt = iv->iv_idt;
set_idtgate(&idt[vec], function, 0, SDT_SYS386IGT, SEL_KPL,
GSEL(GCODE_SEL, SEL_KPL));
}
@@ -207,10 +210,32 @@ idt_vec_set(int vec, void (*function)(vo
* Free IDT vector. No locking required as release is atomic.
*/
void
-idt_vec_free(int vec)
+idt_vec_free(struct idt_vec *iv, int vec)
{
+ idt_descriptor_t *idt;
+ char *idt_allocmap = iv->iv_allocmap;
+ idt = iv->iv_idt;
unset_idtgate(&idt[vec]);
idt_allocmap[vec] = 0;
}
+bool
+idt_vec_is_pcpu(void)
+{
+
+#ifdef PCPU_IDT
+ return true;
+#else
+ return false;
+#endif
+}
+
+struct idt_vec *
+idt_vec_ref(struct idt_vec *iv)
+{
+ if (idt_vec_is_pcpu())
+ return iv;
+
+ return &(cpu_info_primary.ci_idtvec);
+}
Index: src/sys/arch/x86/x86/intr.c
diff -u src/sys/arch/x86/x86/intr.c:1.151 src/sys/arch/x86/x86/intr.c:1.152
--- src/sys/arch/x86/x86/intr.c:1.151 Sat Apr 25 15:26:18 2020
+++ src/sys/arch/x86/x86/intr.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: intr.c,v 1.151 2020/04/25 15:26:18 bouyer Exp $ */
+/* $NetBSD: intr.c,v 1.152 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -133,7 +133,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.151 2020/04/25 15:26:18 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.152 2020/07/14 00:45:53 yamaguchi Exp $");
#include "opt_intrdebug.h"
#include "opt_multiprocessor.h"
@@ -252,12 +252,13 @@ static int intr_set_affinity(struct intr
void
intr_default_setup(void)
{
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
int i;
/* icu vectors */
for (i = 0; i < NUM_LEGACY_IRQS; i++) {
- idt_vec_reserve(ICU_OFFSET + i);
- idt_vec_set(ICU_OFFSET + i, legacy_stubs[i].ist_entry);
+ idt_vec_reserve(iv, ICU_OFFSET + i);
+ idt_vec_set(iv, ICU_OFFSET + i, legacy_stubs[i].ist_entry);
}
/*
@@ -503,6 +504,7 @@ intr_allocate_slot(struct pic *pic, int
struct cpu_info *ci, *lci;
struct intrsource *isp;
int slot = 0, idtvec, error;
+ struct idt_vec *iv;
KASSERT(mutex_owned(&cpu_lock));
@@ -600,7 +602,8 @@ intr_allocate_slot(struct pic *pic, int
* are used by a device using MSI multiple vectors must be
* continuous.
*/
- idtvec = idt_vec_alloc(APIC_LEVEL(level), IDT_INTR_HIGH);
+ iv = idt_vec_ref(&ci->ci_idtvec);
+ idtvec = idt_vec_alloc(iv, APIC_LEVEL(level), IDT_INTR_HIGH);
}
if (idtvec == 0) {
evcnt_detach(&ci->ci_isources[slot]->is_evcnt);
@@ -619,14 +622,16 @@ static void
intr_source_free(struct cpu_info *ci, int slot, struct pic *pic, int idtvec)
{
struct intrsource *isp;
+ struct idt_vec *iv;
isp = ci->ci_isources[slot];
+ iv = idt_vec_ref(&ci->ci_idtvec);
if (isp->is_handlers != NULL)
return;
ci->ci_isources[slot] = NULL;
if (pic != &i8259_pic)
- idt_vec_free(idtvec);
+ idt_vec_free(iv, idtvec);
}
#ifdef MULTIPROCESSOR
@@ -703,6 +708,7 @@ intr_hwunmask_xcall(void *arg1, void *ar
static void
intr_establish_xcall(void *arg1, void *arg2)
{
+ struct idt_vec *iv;
struct intrsource *source;
struct intrstub *stubp;
struct intrhand *ih;
@@ -717,6 +723,7 @@ intr_establish_xcall(void *arg1, void *a
ci = ih->ih_cpu;
source = ci->ci_isources[ih->ih_slot];
idt_vec = (int)(intptr_t)arg2;
+ iv = idt_vec_ref(&ci->ci_idtvec);
/* Disable interrupts locally. */
psl = x86_read_psl();
@@ -729,7 +736,7 @@ intr_establish_xcall(void *arg1, void *a
/* Hook in new IDT vector and SPL state. */
if (source->is_resume == NULL || source->is_idtvec != idt_vec) {
if (source->is_idtvec != 0 && source->is_idtvec != idt_vec)
- idt_vec_free(source->is_idtvec);
+ idt_vec_free(iv, source->is_idtvec);
source->is_idtvec = idt_vec;
if (source->is_type == IST_LEVEL) {
stubp = &source->is_pic->pic_level_stubs[ih->ih_slot];
@@ -738,7 +745,7 @@ intr_establish_xcall(void *arg1, void *a
}
source->is_resume = stubp->ist_resume;
source->is_recurse = stubp->ist_recurse;
- idt_vec_set(idt_vec, stubp->ist_entry);
+ idt_vec_set(iv, idt_vec, stubp->ist_entry);
}
/* Re-enable interrupts locally. */
@@ -1713,6 +1720,7 @@ intr_activate_xcall(void *arg1, void *ar
struct intrsource *source;
struct intrstub *stubp;
struct intrhand *ih;
+ struct idt_vec *iv;
u_long psl;
int idt_vec;
int slot;
@@ -1727,6 +1735,7 @@ intr_activate_xcall(void *arg1, void *ar
slot = ih->ih_slot;
source = ci->ci_isources[slot];
idt_vec = source->is_idtvec;
+ iv = idt_vec_ref(&ci->ci_idtvec);
psl = x86_read_psl();
x86_disable_intr();
@@ -1738,9 +1747,10 @@ intr_activate_xcall(void *arg1, void *ar
} else {
stubp = &source->is_pic->pic_edge_stubs[slot];
}
+
source->is_resume = stubp->ist_resume;
source->is_recurse = stubp->ist_recurse;
- idt_vec_set(idt_vec, stubp->ist_entry);
+ idt_vec_set(iv, idt_vec, stubp->ist_entry);
x86_write_psl(psl);
@@ -1755,7 +1765,9 @@ intr_deactivate_xcall(void *arg1, void *
{
struct cpu_info *ci;
struct intrhand *ih, *lih;
+ struct intrsource *isp;
u_long psl;
+ int idt_vec;
int slot;
ih = arg1;
@@ -1766,6 +1778,8 @@ intr_deactivate_xcall(void *arg1, void *
ci = ih->ih_cpu;
slot = ih->ih_slot;
+ isp = ci->ci_isources[slot];
+ idt_vec = isp->is_idtvec;
psl = x86_read_psl();
x86_disable_intr();
@@ -1778,10 +1792,14 @@ intr_deactivate_xcall(void *arg1, void *
x86_intr_calculatemasks(ci);
- /*
- * Skip unsetgate(), because the same itd[] entry is overwritten in
- * intr_activate_xcall().
- */
+ if (idt_vec > 0 && idt_vec_is_pcpu()) {
+ idt_vec_free(&ci->ci_idtvec, idt_vec);
+ } else {
+ /*
+ * Skip unsetgate(), because the same idt[] entry is
+ * overwritten in intr_activate_xcall().
+ */
+ }
x86_write_psl(psl);
@@ -1820,7 +1838,7 @@ intr_set_affinity(struct intrsource *isp
struct intrhand *ih, *lih;
struct pic *pic;
u_int cpu_idx;
- int idt_vec;
+ int old_idtvec, new_idtvec;
int oldslot, newslot;
int err;
int pin;
@@ -1863,7 +1881,6 @@ intr_set_affinity(struct intrsource *isp
return 0;
oldslot = ih->ih_slot;
- idt_vec = isp->is_idtvec;
err = intr_find_unused_slot(newci, &newslot);
if (err) {
@@ -1872,6 +1889,19 @@ intr_set_affinity(struct intrsource *isp
return err;
}
+ old_idtvec = isp->is_idtvec;
+
+ if (isp->is_idtvec > 0 && idt_vec_is_pcpu()) {
+ new_idtvec = idt_vec_alloc(&newci->ci_idtvec,
+ APIC_LEVEL(ih->ih_level), IDT_INTR_HIGH);
+ if (new_idtvec == 0)
+ return EBUSY;
+ DPRINTF(("interrupt from cpu%d vec %d to cpu%d vec %d\n",
+ cpu_index(oldci), old_idtvec, cpu_index(newci), new_idtvec));
+ } else {
+ new_idtvec = isp->is_idtvec;
+ }
+
/* Prevent intr_unmask() from reenabling the source at the hw. */
isp->is_distribute_pending = true;
@@ -1893,9 +1923,10 @@ intr_set_affinity(struct intrsource *isp
xc_wait(where);
}
intr_save_evcnt(isp, oldci->ci_cpuid);
- (*pic->pic_delroute)(pic, oldci, pin, idt_vec, isp->is_type);
+ (*pic->pic_delroute)(pic, oldci, pin, old_idtvec, isp->is_type);
/* activate new interrupt setting */
+ isp->is_idtvec = new_idtvec;
newci->ci_isources[newslot] = isp;
for (lih = ih; lih != NULL; lih = lih->ih_next) {
newci->ci_nintrhand++;
@@ -1912,7 +1943,7 @@ intr_set_affinity(struct intrsource *isp
}
intr_restore_evcnt(isp, newci->ci_cpuid);
isp->is_active_cpu = newci->ci_cpuid;
- (*pic->pic_addroute)(pic, newci, pin, idt_vec, isp->is_type);
+ (*pic->pic_addroute)(pic, newci, pin, new_idtvec, isp->is_type);
isp->is_distribute_pending = false;
if (newci == curcpu() || !mp_online) {
Index: src/sys/arch/x86/x86/lapic.c
diff -u src/sys/arch/x86/x86/lapic.c:1.83 src/sys/arch/x86/x86/lapic.c:1.84
--- src/sys/arch/x86/x86/lapic.c:1.83 Fri May 29 12:30:41 2020
+++ src/sys/arch/x86/x86/lapic.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lapic.c,v 1.83 2020/05/29 12:30:41 rin Exp $ */
+/* $NetBSD: lapic.c,v 1.84 2020/07/14 00:45:53 yamaguchi Exp $ */
/*-
* Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.83 2020/05/29 12:30:41 rin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.84 2020/07/14 00:45:53 yamaguchi Exp $");
#include "acpica.h"
#include "ioapic.h"
@@ -338,6 +338,8 @@ lapic_setup_bsp(paddr_t lapic_base)
#endif
#if defined(DDB) && defined(MULTIPROCESSOR)
#ifdef __x86_64__
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
+ idt_descriptor_t *idt = iv->iv_idt;
set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
#else
@@ -475,23 +477,24 @@ lapic_set_lvt(void)
void
lapic_boot_init(paddr_t lapic_base)
{
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
lapic_setup_bsp(lapic_base);
#ifdef MULTIPROCESSOR
- idt_vec_reserve(LAPIC_IPI_VECTOR);
- idt_vec_set(LAPIC_IPI_VECTOR,
+ idt_vec_reserve(iv, LAPIC_IPI_VECTOR);
+ idt_vec_set(iv, LAPIC_IPI_VECTOR,
x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi);
- idt_vec_reserve(LAPIC_TLB_VECTOR);
- idt_vec_set(LAPIC_TLB_VECTOR,
+ idt_vec_reserve(iv, LAPIC_TLB_VECTOR);
+ idt_vec_set(iv, LAPIC_TLB_VECTOR,
x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb);
#endif
- idt_vec_reserve(LAPIC_SPURIOUS_VECTOR);
- idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious);
+ idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR);
+ idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious);
- idt_vec_reserve(LAPIC_TIMER_VECTOR);
- idt_vec_set(LAPIC_TIMER_VECTOR,
+ idt_vec_reserve(iv, LAPIC_TIMER_VECTOR);
+ idt_vec_set(iv, LAPIC_TIMER_VECTOR,
x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer);
}
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.399 src/sys/arch/x86/x86/pmap.c:1.400
--- src/sys/arch/x86/x86/pmap.c:1.399 Sun Jun 14 21:41:42 2020
+++ src/sys/arch/x86/x86/pmap.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.399 2020/06/14 21:41:42 ad Exp $ */
+/* $NetBSD: pmap.c,v 1.400 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017, 2019, 2020 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.399 2020/06/14 21:41:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.400 2020/07/14 00:45:53 yamaguchi Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -1309,11 +1309,7 @@ pmap_bootstrap(vaddr_t kva_start)
/*
* Allocate space for the IDT, GDT and LDT.
*/
-#ifdef __HAVE_PCPU_AREA
- idt_vaddr = (vaddr_t)&pcpuarea->idt;
-#else
idt_vaddr = pmap_bootstrap_valloc(1);
-#endif
idt_paddr = pmap_bootstrap_palloc(1);
gdt_vaddr = pmap_bootstrap_valloc(1);
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.37 src/sys/arch/x86/x86/svs.c:1.38
--- src/sys/arch/x86/x86/svs.c:1.37 Wed May 27 19:40:29 2020
+++ src/sys/arch/x86/x86/svs.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.37 2020/05/27 19:40:29 ad Exp $ */
+/* $NetBSD: svs.c,v 1.38 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.37 2020/05/27 19:40:29 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.38 2020/07/14 00:45:53 yamaguchi Exp $");
#include "opt_svs.h"
#include "opt_user_ldt.h"
@@ -519,6 +519,7 @@ cpu_svs_init(struct cpu_info *ci)
{
extern char __text_user_start;
extern char __text_user_end;
+ extern vaddr_t idt_vaddr;
const cpuid_t cid = cpu_index(ci);
struct vm_page *pg;
@@ -543,7 +544,8 @@ cpu_svs_init(struct cpu_info *ci)
mutex_init(&ci->ci_svs_mtx, MUTEX_DEFAULT, IPL_VM);
- svs_page_add(ci, (vaddr_t)&pcpuarea->idt, true);
+ if (cid == cpu_index(&cpu_info_primary) || !idt_vec_is_pcpu())
+ svs_page_add(ci, idt_vaddr, true);
svs_page_add(ci, (vaddr_t)&pcpuarea->ldt, true);
svs_range_add(ci, (vaddr_t)&pcpuarea->ent[cid],
offsetof(struct pcpu_entry, rsp0), true);
Index: src/sys/arch/xen/x86/cpu.c
diff -u src/sys/arch/xen/x86/cpu.c:1.138 src/sys/arch/xen/x86/cpu.c:1.139
--- src/sys/arch/xen/x86/cpu.c:1.138 Wed Jul 8 11:11:00 2020
+++ src/sys/arch/xen/x86/cpu.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.138 2020/07/08 11:11:00 jdolecek Exp $ */
+/* $NetBSD: cpu.c,v 1.139 2020/07/14 00:45:53 yamaguchi Exp $ */
/*-
* Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -65,7 +65,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.138 2020/07/08 11:11:00 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.139 2020/07/14 00:45:53 yamaguchi Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -477,6 +477,9 @@ cpu_attach_common(device_t parent, devic
/* interrupt handler stack */
cpu_intr_init(ci);
+ /* Setup per-cpu memory for idt */
+ idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci));
+
/* Setup per-cpu memory for gdt */
gdt_alloc_cpu(ci);
@@ -700,7 +703,7 @@ cpu_hatch(void *v)
/* Setup TLS and kernel GS/FS */
cpu_init_msrs(ci, true);
- cpu_init_idt();
+ cpu_init_idt(ci);
gdt_init_cpu(ci);
cpu_probe(ci);
Index: src/sys/arch/xen/xen/hypervisor.c
diff -u src/sys/arch/xen/xen/hypervisor.c:1.86 src/sys/arch/xen/xen/hypervisor.c:1.87
--- src/sys/arch/xen/xen/hypervisor.c:1.86 Tue May 26 10:37:25 2020
+++ src/sys/arch/xen/xen/hypervisor.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: hypervisor.c,v 1.86 2020/05/26 10:37:25 bouyer Exp $ */
+/* $NetBSD: hypervisor.c,v 1.87 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 2005 Manuel Bouyer.
@@ -53,7 +53,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hypervisor.c,v 1.86 2020/05/26 10:37:25 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hypervisor.c,v 1.87 2020/07/14 00:45:53 yamaguchi Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -295,6 +295,8 @@ xen_check_hypervisordev(void)
static int
xen_hvm_init_late(void)
{
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
+
if (HYPERVISOR_xen_version(XENVER_version, NULL) < 0) {
aprint_error("Xen HVM: hypercall page not working\n");
return 0;
@@ -350,8 +352,8 @@ xen_hvm_init_late(void)
* prepare vector.
* We don't really care where it is, as long as it's free
*/
- xen_hvm_vec = idt_vec_alloc(129, 255);
- idt_vec_set(xen_hvm_vec, &IDTVEC(hypervisor_pvhvm_callback));
+ xen_hvm_vec = idt_vec_alloc(iv, 129, 255);
+ idt_vec_set(iv, xen_hvm_vec, &IDTVEC(hypervisor_pvhvm_callback));
events_default_setup();
return 1;
Index: src/sys/dev/hyperv/vmbusvar.h
diff -u src/sys/dev/hyperv/vmbusvar.h:1.5 src/sys/dev/hyperv/vmbusvar.h:1.6
--- src/sys/dev/hyperv/vmbusvar.h:1.5 Tue May 26 16:08:55 2020
+++ src/sys/dev/hyperv/vmbusvar.h Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vmbusvar.h,v 1.5 2020/05/26 16:08:55 nonaka Exp $ */
+/* $NetBSD: vmbusvar.h,v 1.6 2020/07/14 00:45:53 yamaguchi Exp $ */
/* $OpenBSD: hypervvar.h,v 1.13 2017/06/23 19:05:42 mikeb Exp $ */
/*
@@ -159,6 +159,8 @@ struct vmbus_percpu_data {
/* Rarely used fields */
struct hyperv_dma simp_dma;
struct hyperv_dma siep_dma;
+
+ void *md_cookie;
} __aligned(CACHE_LINE_SIZE);
struct vmbus_softc {
Index: src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.61 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.62
--- src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.61 Fri Jul 3 16:09:54 2020
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmx.c Tue Jul 14 00:45:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmx.c,v 1.61 2020/07/03 16:09:54 maxv Exp $ */
+/* $NetBSD: nvmm_x86_vmx.c,v 1.62 2020/07/14 00:45:53 yamaguchi Exp $ */
/*
* Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.61 2020/07/03 16:09:54 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.62 2020/07/14 00:45:53 yamaguchi Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -2698,7 +2698,8 @@ vmx_vcpu_init(struct nvmm_machine *mach,
struct vmcs *vmcs = cpudata->vmcs;
struct msr_entry *gmsr = cpudata->gmsr;
extern uint8_t vmx_resume_rip;
- uint64_t rev, eptp;
+ uint64_t rev, eptp, idt;
+ struct cpu_info *ci;
rev = vmx_get_revision();
@@ -2765,6 +2766,9 @@ vmx_vcpu_init(struct nvmm_machine *mach,
vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE);
/* Set the Host state for resuming. */
+ ci = curcpu();
+ idt = (uint64_t)ci->ci_idtvec.iv_idt;
+
vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip);
vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL));
vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
@@ -2775,7 +2779,7 @@ vmx_vcpu_init(struct nvmm_machine *mach,
vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0);
vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0);
vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0);
- vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt);
+ vmx_vmwrite(VMCS_HOST_IDTR_BASE, idt);
vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT));
vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER));
vmx_vmwrite(VMCS_HOST_CR0, rcr0() & ~CR0_TS);
