Module Name: src Committed By: maxv Date: Thu Jul 12 19:48:16 UTC 2018
Modified Files: src/sys/arch/amd64/amd64: amd64_trap.S locore.S machdep.c src/sys/arch/amd64/include: frameasm.h src/sys/arch/x86/x86: svs.c Log Message: Handle NMIs correctly when SVS is enabled. We store the kernel's CR3 at the top of the NMI stack, and we unconditionally switch to it, because we don't know with which page tables we received the NMI. Hotpatch the whole thing as usual. This restores the ability to use PMCs on Intel CPUs. To generate a diff of this commit: cvs rdiff -u -r1.42 -r1.43 src/sys/arch/amd64/amd64/amd64_trap.S cvs rdiff -u -r1.169 -r1.170 src/sys/arch/amd64/amd64/locore.S cvs rdiff -u -r1.305 -r1.306 src/sys/arch/amd64/amd64/machdep.c cvs rdiff -u -r1.38 -r1.39 src/sys/arch/amd64/include/frameasm.h cvs rdiff -u -r1.18 -r1.19 src/sys/arch/x86/x86/svs.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/amd64/amd64/amd64_trap.S diff -u src/sys/arch/amd64/amd64/amd64_trap.S:1.42 src/sys/arch/amd64/amd64/amd64_trap.S:1.43 --- src/sys/arch/amd64/amd64/amd64_trap.S:1.42 Fri May 25 15:33:56 2018 +++ src/sys/arch/amd64/amd64/amd64_trap.S Thu Jul 12 19:48:16 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: amd64_trap.S,v 1.42 2018/05/25 15:33:56 maxv Exp $ */ +/* $NetBSD: amd64_trap.S,v 1.43 2018/07/12 19:48:16 maxv Exp $ */ /* * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc. @@ -254,7 +254,6 @@ IDTVEC(trap02) subq $TF_REGSIZE,%rsp INTR_SAVE_GPRS IBRS_ENTER - SVS_ENTER_ALTSTACK cld SMAP_ENABLE movw %gs,TF_GS(%rsp) @@ -262,6 +261,8 @@ IDTVEC(trap02) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) + SVS_ENTER_NMI + movl $MSR_GSBASE,%ecx rdmsr cmpl $VM_MIN_KERNEL_ADDRESS_HIGH32,%edx @@ -271,7 +272,6 @@ IDTVEC(trap02) movq %rsp,%rdi incq CPUVAR(NTRAP) call _C_LABEL(nmitrap) - SVS_LEAVE_ALTSTACK swapgs jmp .Lnmileave @@ -279,9 +279,9 @@ IDTVEC(trap02) movq %rsp,%rdi incq CPUVAR(NTRAP) call _C_LABEL(nmitrap) - SVS_LEAVE_ALTSTACK .Lnmileave: + SVS_LEAVE_NMI IBRS_LEAVE INTR_RESTORE_GPRS addq $TF_REGSIZE+16,%rsp Index: src/sys/arch/amd64/amd64/locore.S diff -u src/sys/arch/amd64/amd64/locore.S:1.169 src/sys/arch/amd64/amd64/locore.S:1.170 --- src/sys/arch/amd64/amd64/locore.S:1.169 Mon Jul 9 18:52:04 2018 +++ src/sys/arch/amd64/amd64/locore.S Thu Jul 12 19:48:16 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: locore.S,v 1.169 2018/07/09 18:52:04 maxv Exp $ */ +/* $NetBSD: locore.S,v 1.170 2018/07/12 19:48:16 maxv Exp $ */ /* * Copyright-o-rama! @@ -1633,6 +1633,13 @@ LABEL(svs_enter_altstack) 1234: LABEL(svs_enter_altstack_end) +LABEL(svs_enter_nmi) + movq %cr3,%rax + movq %rax,(FRAMESIZE+1*8)(%rsp) /* nmistore->scratch */ + movq (FRAMESIZE+0*8)(%rsp),%rax /* nmistore->cr3 */ + movq %rax,%cr3 +LABEL(svs_enter_nmi_end) + LABEL(svs_leave) testb $SEL_UPL,TF_CS(%rsp) jz 1234f @@ -1650,6 +1657,11 @@ LABEL(svs_leave_altstack) 1234: LABEL(svs_leave_altstack_end) +LABEL(svs_leave_nmi) + movq (FRAMESIZE+1*8)(%rsp),%rax /* nmistore->scratch */ + movq %rax,%cr3 +LABEL(svs_leave_nmi_end) + LABEL(nosvs_enter) NOSVS_ENTER LABEL(nosvs_enter_end) @@ -1658,6 +1670,10 @@ LABEL(nosvs_enter_altstack) NOSVS_ENTER_ALTSTACK LABEL(nosvs_enter_altstack_end) +LABEL(nosvs_enter_nmi) + NOSVS_ENTER_NMI +LABEL(nosvs_enter_nmi_end) + LABEL(nosvs_leave) NOSVS_LEAVE LABEL(nosvs_leave_end) @@ -1665,6 +1681,10 @@ LABEL(nosvs_leave_end) LABEL(nosvs_leave_altstack) NOSVS_LEAVE_ALTSTACK LABEL(nosvs_leave_altstack_end) + +LABEL(nosvs_leave_nmi) + NOSVS_LEAVE_NMI +LABEL(nosvs_leave_nmi_end) #endif .globl ibrs_enter, ibrs_enter_end Index: src/sys/arch/amd64/amd64/machdep.c diff -u src/sys/arch/amd64/amd64/machdep.c:1.305 src/sys/arch/amd64/amd64/machdep.c:1.306 --- src/sys/arch/amd64/amd64/machdep.c:1.305 Wed Jun 20 11:49:37 2018 +++ src/sys/arch/amd64/amd64/machdep.c Thu Jul 12 19:48:16 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: machdep.c,v 1.305 2018/06/20 11:49:37 maxv Exp $ */ +/* $NetBSD: machdep.c,v 1.306 2018/07/12 19:48:16 maxv Exp $ */ /* * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011 @@ -110,7 +110,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.305 2018/06/20 11:49:37 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.306 2018/07/12 19:48:16 maxv Exp $"); #include "opt_modular.h" #include "opt_user_ldt.h" @@ -281,6 +281,11 @@ void (*initclock_func)(void) = xen_initc struct pool x86_dbregspl; +struct nmistore { + uint64_t cr3; + uint64_t scratch; +} __packed; + /* * Size of memory segments, before any memory is stolen. */ @@ -506,6 +511,7 @@ cpu_init_tss(struct cpu_info *ci) const cpuid_t cid = cpu_index(ci); #endif struct cpu_tss *cputss; + struct nmistore *store; uintptr_t p; #ifdef __HAVE_PCPU_AREA @@ -533,13 +539,15 @@ cpu_init_tss(struct cpu_info *ci) #endif cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16; - /* NMI */ + /* NMI - store a structure at the top of the stack */ #ifdef __HAVE_PCPU_AREA p = (vaddr_t)&pcpuarea->ent[cid].ist2; #else p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); #endif - cputss->tss.tss_ist[2] = p + PAGE_SIZE - 16; + cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore); + store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore)); + store->cr3 = pmap_pdirpa(pmap_kernel(), 0); /* DB */ #ifdef __HAVE_PCPU_AREA Index: src/sys/arch/amd64/include/frameasm.h diff -u src/sys/arch/amd64/include/frameasm.h:1.38 src/sys/arch/amd64/include/frameasm.h:1.39 --- src/sys/arch/amd64/include/frameasm.h:1.38 Wed Mar 28 16:02:49 2018 +++ src/sys/arch/amd64/include/frameasm.h Thu Jul 12 19:48:16 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $ */ +/* $NetBSD: frameasm.h,v 1.39 2018/07/12 19:48:16 maxv Exp $ */ #ifndef _AMD64_MACHINE_FRAMEASM_H #define _AMD64_MACHINE_FRAMEASM_H @@ -46,6 +46,8 @@ #define HP_NAME_SVS_LEAVE_ALT 8 #define HP_NAME_IBRS_ENTER 9 #define HP_NAME_IBRS_LEAVE 10 +#define HP_NAME_SVS_ENTER_NMI 11 +#define HP_NAME_SVS_LEAVE_NMI 12 #define HOTPATCH(name, size) \ 123: ; \ @@ -165,6 +167,22 @@ HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \ NOSVS_LEAVE_ALTSTACK +#define SVS_ENTER_NMI_BYTES 22 +#define NOSVS_ENTER_NMI \ + .byte 0xEB, (SVS_ENTER_NMI_BYTES-2) /* jmp */ ; \ + .fill (SVS_ENTER_NMI_BYTES-2),1,0xCC +#define SVS_ENTER_NMI \ + HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES) ; \ + NOSVS_ENTER_NMI + +#define SVS_LEAVE_NMI_BYTES 11 +#define NOSVS_LEAVE_NMI \ + .byte 0xEB, (SVS_LEAVE_NMI_BYTES-2) /* jmp */ ; \ + .fill (SVS_LEAVE_NMI_BYTES-2),1,0xCC +#define SVS_LEAVE_NMI \ + HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES) ; \ + NOSVS_LEAVE_NMI + #else #define SVS_ENTER /* nothing */ #define SVS_LEAVE /* nothing */ Index: src/sys/arch/x86/x86/svs.c diff -u src/sys/arch/x86/x86/svs.c:1.18 src/sys/arch/x86/x86/svs.c:1.19 --- src/sys/arch/x86/x86/svs.c:1.18 Thu Apr 26 18:54:09 2018 +++ src/sys/arch/x86/x86/svs.c Thu Jul 12 19:48:16 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: svs.c,v 1.18 2018/04/26 18:54:09 alnsn Exp $ */ +/* $NetBSD: svs.c,v 1.19 2018/07/12 19:48:16 maxv Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.18 2018/04/26 18:54:09 alnsn Exp $"); +__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.19 2018/07/12 19:48:16 maxv Exp $"); #include "opt_svs.h" @@ -571,8 +571,10 @@ svs_enable(void) { extern uint8_t svs_enter, svs_enter_end; extern uint8_t svs_enter_altstack, svs_enter_altstack_end; + extern uint8_t svs_enter_nmi, svs_enter_nmi_end; extern uint8_t svs_leave, svs_leave_end; extern uint8_t svs_leave_altstack, svs_leave_altstack_end; + extern uint8_t svs_leave_nmi, svs_leave_nmi_end; u_long psl, cr0; uint8_t *bytes; size_t size; @@ -590,6 +592,11 @@ svs_enable(void) (size_t)&svs_enter_altstack; x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size); + bytes = &svs_enter_nmi; + size = (size_t)&svs_enter_nmi_end - + (size_t)&svs_enter_nmi; + x86_hotpatch(HP_NAME_SVS_ENTER_NMI, bytes, size); + bytes = &svs_leave; size = (size_t)&svs_leave_end - (size_t)&svs_leave; x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size); @@ -599,6 +606,11 @@ svs_enable(void) (size_t)&svs_leave_altstack; x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size); + bytes = &svs_leave_nmi; + size = (size_t)&svs_leave_nmi_end - + (size_t)&svs_leave_nmi; + x86_hotpatch(HP_NAME_SVS_LEAVE_NMI, bytes, size); + x86_patch_window_close(psl, cr0); } @@ -607,8 +619,10 @@ svs_disable_hotpatch(void) { extern uint8_t nosvs_enter, nosvs_enter_end; extern uint8_t nosvs_enter_altstack, nosvs_enter_altstack_end; + extern uint8_t nosvs_enter_nmi, nosvs_enter_nmi_end; extern uint8_t nosvs_leave, nosvs_leave_end; extern uint8_t nosvs_leave_altstack, nosvs_leave_altstack_end; + extern uint8_t nosvs_leave_nmi, nosvs_leave_nmi_end; u_long psl, cr0; uint8_t *bytes; size_t size; @@ -624,6 +638,11 @@ svs_disable_hotpatch(void) (size_t)&nosvs_enter_altstack; x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size); + bytes = &nosvs_enter_nmi; + size = (size_t)&nosvs_enter_nmi_end - + (size_t)&nosvs_enter_nmi; + x86_hotpatch(HP_NAME_SVS_ENTER_NMI, bytes, size); + bytes = &nosvs_leave; size = (size_t)&nosvs_leave_end - (size_t)&nosvs_leave; x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size); @@ -633,6 +652,11 @@ svs_disable_hotpatch(void) (size_t)&nosvs_leave_altstack; x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size); + bytes = &nosvs_leave_nmi; + size = (size_t)&nosvs_leave_nmi_end - + (size_t)&nosvs_leave_nmi; + x86_hotpatch(HP_NAME_SVS_LEAVE_NMI, bytes, size); + x86_patch_window_close(psl, cr0); }