Module Name: src Committed By: maxv Date: Thu Feb 22 09:41:06 UTC 2018
Modified Files: src/sys/arch/amd64/amd64: machdep.c src/sys/arch/amd64/include: pmap.h src/sys/arch/x86/include: cpufunc.h src/sys/arch/x86/x86: patch.c svs.c x86_machdep.c Log Message: Improve the SVS initialization. Declare x86_patch_window_open() and x86_patch_window_close(), and globalify x86_hotpatch(). Introduce svs_enable() in x86/svs.c, that does the SVS hotpatching. Change svs_init() to take a bool. This function gets called twice; early when the system just booted (and nothing is initialized), lately when at least pmap_kernel has been initialized. To generate a diff of this commit: cvs rdiff -u -r1.298 -r1.299 src/sys/arch/amd64/amd64/machdep.c cvs rdiff -u -r1.43 -r1.44 src/sys/arch/amd64/include/pmap.h cvs rdiff -u -r1.23 -r1.24 src/sys/arch/x86/include/cpufunc.h cvs rdiff -u -r1.32 -r1.33 src/sys/arch/x86/x86/patch.c cvs rdiff -u -r1.4 -r1.5 src/sys/arch/x86/x86/svs.c cvs rdiff -u -r1.104 -r1.105 src/sys/arch/x86/x86/x86_machdep.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/amd64/amd64/machdep.c diff -u src/sys/arch/amd64/amd64/machdep.c:1.298 src/sys/arch/amd64/amd64/machdep.c:1.299 --- src/sys/arch/amd64/amd64/machdep.c:1.298 Sun Feb 11 09:39:36 2018 +++ src/sys/arch/amd64/amd64/machdep.c Thu Feb 22 09:41:06 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: machdep.c,v 1.298 2018/02/11 09:39:36 maxv Exp $ */ +/* $NetBSD: machdep.c,v 1.299 2018/02/22 09:41:06 maxv Exp $ */ /* * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011 @@ -110,7 +110,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.298 2018/02/11 09:39:36 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.299 2018/02/22 09:41:06 maxv Exp $"); /* #define XENDEBUG_LOW */ @@ -1598,6 +1598,7 @@ init_x86_64(paddr_t first_avail) uvm_lwp_setuarea(&lwp0, lwp0uarea); cpu_probe(&cpu_info_primary); + svs_init(true); cpu_init_msrs(&cpu_info_primary, true); use_pae = 1; /* PAE always enabled in long mode */ Index: src/sys/arch/amd64/include/pmap.h diff -u src/sys/arch/amd64/include/pmap.h:1.43 src/sys/arch/amd64/include/pmap.h:1.44 --- src/sys/arch/amd64/include/pmap.h:1.43 Sun Feb 18 14:07:29 2018 +++ src/sys/arch/amd64/include/pmap.h Thu Feb 22 09:41:06 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.h,v 1.43 2018/02/18 14:07:29 maxv Exp $ */ +/* $NetBSD: pmap.h,v 1.44 2018/02/22 09:41:06 maxv Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -221,6 +221,7 @@ void svs_pmap_sync(struct pmap *, int); void svs_lwp_switch(struct lwp *, struct lwp *); void svs_pdir_switch(struct pmap *); +void svs_init(bool); extern bool svs_enabled; #include <x86/pmap.h> Index: src/sys/arch/x86/include/cpufunc.h diff -u src/sys/arch/x86/include/cpufunc.h:1.23 src/sys/arch/x86/include/cpufunc.h:1.24 --- src/sys/arch/x86/include/cpufunc.h:1.23 Sun Oct 15 11:31:00 2017 +++ src/sys/arch/x86/include/cpufunc.h Thu Feb 22 09:41:06 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: cpufunc.h,v 1.23 2017/10/15 11:31:00 maxv Exp $ */ +/* $NetBSD: cpufunc.h,v 1.24 2018/02/22 09:41:06 maxv Exp $ */ /*- * Copyright (c) 1998, 2007 The NetBSD Foundation, Inc. @@ -50,6 +50,9 @@ void x86_sfence(void); void x86_mfence(void); void x86_flush(void); #ifndef XEN +void x86_hotpatch(uint32_t, const uint8_t *, size_t); +void x86_patch_window_open(u_long *, u_long *); +void x86_patch_window_close(u_long, u_long); void x86_patch(bool); #endif void invlpg(vaddr_t); Index: src/sys/arch/x86/x86/patch.c diff -u src/sys/arch/x86/x86/patch.c:1.32 src/sys/arch/x86/x86/patch.c:1.33 --- src/sys/arch/x86/x86/patch.c:1.32 Thu Feb 22 08:56:52 2018 +++ src/sys/arch/x86/x86/patch.c Thu Feb 22 09:41:06 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: patch.c,v 1.32 2018/02/22 08:56:52 maxv Exp $ */ +/* $NetBSD: patch.c,v 1.33 2018/02/22 09:41:06 maxv Exp $ */ /*- * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. @@ -34,7 +34,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.32 2018/02/22 08:56:52 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.33 2018/02/22 09:41:06 maxv Exp $"); #include "opt_lockdebug.h" #ifdef i386 @@ -143,7 +143,7 @@ patchbytes(void *addr, const uint8_t *by } } -static void +void x86_hotpatch(uint32_t name, const uint8_t *bytes, size_t size) { extern char __rodata_hotpatch_start; @@ -165,6 +165,30 @@ x86_hotpatch(uint32_t name, const uint8_ } void +x86_patch_window_open(u_long *psl, u_long *cr0) +{ + /* Disable interrupts. */ + *psl = x86_read_psl(); + x86_disable_intr(); + + /* Disable write protection in supervisor mode. */ + *cr0 = rcr0(); + lcr0(*cr0 & ~CR0_WP); +} + +void +x86_patch_window_close(u_long psl, u_long cr0) +{ + /* Write back and invalidate cache, flush pipelines. */ + wbinvd(); + x86_flush(); + x86_write_psl(psl); + + /* Re-enable write protection. */ + lcr0(cr0); +} + +void x86_patch(bool early) { static bool first, second; @@ -181,13 +205,7 @@ x86_patch(bool early) second = true; } - /* Disable interrupts. */ - psl = x86_read_psl(); - x86_disable_intr(); - - /* Disable write protection in supervisor mode. */ - cr0 = rcr0(); - lcr0(cr0 & ~CR0_WP); + x86_patch_window_open(&psl, &cr0); #if !defined(GPROF) if (!early && ncpu == 1) { @@ -298,43 +316,5 @@ x86_patch(bool early) x86_hotpatch(HP_NAME_STAC, stac_bytes, sizeof(stac_bytes)); } -#ifdef SVS - if (early && cpu_vendor == CPUVENDOR_INTEL) { - extern uint8_t svs_enter, svs_enter_end; - extern uint8_t svs_enter_altstack, svs_enter_altstack_end; - extern uint8_t svs_leave, svs_leave_end; - extern uint8_t svs_leave_altstack, svs_leave_altstack_end; - extern bool svs_enabled; - uint8_t *bytes; - size_t size; - - svs_enabled = true; - - bytes = &svs_enter; - size = (size_t)&svs_enter_end - (size_t)&svs_enter; - x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size); - - bytes = &svs_enter_altstack; - size = (size_t)&svs_enter_altstack_end - - (size_t)&svs_enter_altstack; - x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size); - - bytes = &svs_leave; - size = (size_t)&svs_leave_end - (size_t)&svs_leave; - x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size); - - bytes = &svs_leave_altstack; - size = (size_t)&svs_leave_altstack_end - - (size_t)&svs_leave_altstack; - x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size); - } -#endif - - /* Write back and invalidate cache, flush pipelines. */ - wbinvd(); - x86_flush(); - x86_write_psl(psl); - - /* Re-enable write protection. */ - lcr0(cr0); + x86_patch_window_close(psl, cr0); } Index: src/sys/arch/x86/x86/svs.c diff -u src/sys/arch/x86/x86/svs.c:1.4 src/sys/arch/x86/x86/svs.c:1.5 --- src/sys/arch/x86/x86/svs.c:1.4 Thu Feb 22 08:56:52 2018 +++ src/sys/arch/x86/x86/svs.c Thu Feb 22 09:41:06 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: svs.c,v 1.4 2018/02/22 08:56:52 maxv Exp $ */ +/* $NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.4 2018/02/22 08:56:52 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $"); #include "opt_svs.h" @@ -39,7 +39,9 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.4 #include <sys/proc.h> #include <sys/cpu.h> +#include <x86/cputypes.h> #include <machine/cpuvar.h> +#include <machine/frameasm.h> #include <uvm/uvm.h> #include <uvm/uvm_page.h> @@ -503,12 +505,56 @@ svs_pgg_update(bool enable) tlbflushg(); } -void svs_init(void); +static void +svs_enable(void) +{ + extern uint8_t svs_enter, svs_enter_end; + extern uint8_t svs_enter_altstack, svs_enter_altstack_end; + extern uint8_t svs_leave, svs_leave_end; + extern uint8_t svs_leave_altstack, svs_leave_altstack_end; + u_long psl, cr0; + uint8_t *bytes; + size_t size; + + svs_enabled = true; + + x86_patch_window_open(&psl, &cr0); + + bytes = &svs_enter; + size = (size_t)&svs_enter_end - (size_t)&svs_enter; + x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size); + + bytes = &svs_enter_altstack; + size = (size_t)&svs_enter_altstack_end - + (size_t)&svs_enter_altstack; + x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size); + + bytes = &svs_leave; + size = (size_t)&svs_leave_end - (size_t)&svs_leave; + x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size); + + bytes = &svs_leave_altstack; + size = (size_t)&svs_leave_altstack_end - + (size_t)&svs_leave_altstack; + x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size); + + x86_patch_window_close(psl, cr0); +} void -svs_init(void) +svs_init(bool early) { - if (svs_enabled) + /* + * When early, declare that we want to use SVS, and hotpatch the + * entry points. When late, remove PG_G from the page tables. + */ + if (early) { + if (cpu_vendor != CPUVENDOR_INTEL) { + return; + } + svs_enable(); + } else if (svs_enabled) { svs_pgg_update(false); + } } Index: src/sys/arch/x86/x86/x86_machdep.c diff -u src/sys/arch/x86/x86/x86_machdep.c:1.104 src/sys/arch/x86/x86/x86_machdep.c:1.105 --- src/sys/arch/x86/x86/x86_machdep.c:1.104 Thu Feb 22 08:56:52 2018 +++ src/sys/arch/x86/x86/x86_machdep.c Thu Feb 22 09:41:06 2018 @@ -1,4 +1,4 @@ -/* $NetBSD: x86_machdep.c,v 1.104 2018/02/22 08:56:52 maxv Exp $ */ +/* $NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, @@ -31,7 +31,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.104 2018/02/22 08:56:52 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $"); #include "opt_modular.h" #include "opt_physmem.h" @@ -1092,8 +1092,7 @@ void x86_startup(void) { #if SVS - void svs_init(void); - svs_init(); + svs_init(false); #endif #if !defined(XEN) nmi_init();