Module Name: src
Committed By: maxv
Date: Thu Feb 22 09:41:06 UTC 2018
Modified Files:
src/sys/arch/amd64/amd64: machdep.c
src/sys/arch/amd64/include: pmap.h
src/sys/arch/x86/include: cpufunc.h
src/sys/arch/x86/x86: patch.c svs.c x86_machdep.c
Log Message:
Improve the SVS initialization.
Declare x86_patch_window_open() and x86_patch_window_close(), and globalify
x86_hotpatch().
Introduce svs_enable() in x86/svs.c, that does the SVS hotpatching.
Change svs_init() to take a bool. This function gets called twice; early
when the system just booted (and nothing is initialized), lately when at
least pmap_kernel has been initialized.
To generate a diff of this commit:
cvs rdiff -u -r1.298 -r1.299 src/sys/arch/amd64/amd64/machdep.c
cvs rdiff -u -r1.43 -r1.44 src/sys/arch/amd64/include/pmap.h
cvs rdiff -u -r1.23 -r1.24 src/sys/arch/x86/include/cpufunc.h
cvs rdiff -u -r1.32 -r1.33 src/sys/arch/x86/x86/patch.c
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/x86/x86/svs.c
cvs rdiff -u -r1.104 -r1.105 src/sys/arch/x86/x86/x86_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/machdep.c
diff -u src/sys/arch/amd64/amd64/machdep.c:1.298 src/sys/arch/amd64/amd64/machdep.c:1.299
--- src/sys/arch/amd64/amd64/machdep.c:1.298 Sun Feb 11 09:39:36 2018
+++ src/sys/arch/amd64/amd64/machdep.c Thu Feb 22 09:41:06 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.298 2018/02/11 09:39:36 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.299 2018/02/22 09:41:06 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.298 2018/02/11 09:39:36 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.299 2018/02/22 09:41:06 maxv Exp $");
/* #define XENDEBUG_LOW */
@@ -1598,6 +1598,7 @@ init_x86_64(paddr_t first_avail)
uvm_lwp_setuarea(&lwp0, lwp0uarea);
cpu_probe(&cpu_info_primary);
+ svs_init(true);
cpu_init_msrs(&cpu_info_primary, true);
use_pae = 1; /* PAE always enabled in long mode */
Index: src/sys/arch/amd64/include/pmap.h
diff -u src/sys/arch/amd64/include/pmap.h:1.43 src/sys/arch/amd64/include/pmap.h:1.44
--- src/sys/arch/amd64/include/pmap.h:1.43 Sun Feb 18 14:07:29 2018
+++ src/sys/arch/amd64/include/pmap.h Thu Feb 22 09:41:06 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.43 2018/02/18 14:07:29 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.44 2018/02/22 09:41:06 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -221,6 +221,7 @@
void svs_pmap_sync(struct pmap *, int);
void svs_lwp_switch(struct lwp *, struct lwp *);
void svs_pdir_switch(struct pmap *);
+void svs_init(bool);
extern bool svs_enabled;
#include <x86/pmap.h>
Index: src/sys/arch/x86/include/cpufunc.h
diff -u src/sys/arch/x86/include/cpufunc.h:1.23 src/sys/arch/x86/include/cpufunc.h:1.24
--- src/sys/arch/x86/include/cpufunc.h:1.23 Sun Oct 15 11:31:00 2017
+++ src/sys/arch/x86/include/cpufunc.h Thu Feb 22 09:41:06 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.23 2017/10/15 11:31:00 maxv Exp $ */
+/* $NetBSD: cpufunc.h,v 1.24 2018/02/22 09:41:06 maxv Exp $ */
/*-
* Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
@@ -50,6 +50,9 @@ void x86_sfence(void);
void x86_mfence(void);
void x86_flush(void);
#ifndef XEN
+void x86_hotpatch(uint32_t, const uint8_t *, size_t);
+void x86_patch_window_open(u_long *, u_long *);
+void x86_patch_window_close(u_long, u_long);
void x86_patch(bool);
#endif
void invlpg(vaddr_t);
Index: src/sys/arch/x86/x86/patch.c
diff -u src/sys/arch/x86/x86/patch.c:1.32 src/sys/arch/x86/x86/patch.c:1.33
--- src/sys/arch/x86/x86/patch.c:1.32 Thu Feb 22 08:56:52 2018
+++ src/sys/arch/x86/x86/patch.c Thu Feb 22 09:41:06 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: patch.c,v 1.32 2018/02/22 08:56:52 maxv Exp $ */
+/* $NetBSD: patch.c,v 1.33 2018/02/22 09:41:06 maxv Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.32 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.33 2018/02/22 09:41:06 maxv Exp $");
#include "opt_lockdebug.h"
#ifdef i386
@@ -143,7 +143,7 @@ patchbytes(void *addr, const uint8_t *by
}
}
-static void
+void
x86_hotpatch(uint32_t name, const uint8_t *bytes, size_t size)
{
extern char __rodata_hotpatch_start;
@@ -165,6 +165,30 @@ x86_hotpatch(uint32_t name, const uint8_
}
void
+x86_patch_window_open(u_long *psl, u_long *cr0)
+{
+ /* Disable interrupts. */
+ *psl = x86_read_psl();
+ x86_disable_intr();
+
+ /* Disable write protection in supervisor mode. */
+ *cr0 = rcr0();
+ lcr0(*cr0 & ~CR0_WP);
+}
+
+void
+x86_patch_window_close(u_long psl, u_long cr0)
+{
+ /* Write back and invalidate cache, flush pipelines. */
+ wbinvd();
+ x86_flush();
+ x86_write_psl(psl);
+
+ /* Re-enable write protection. */
+ lcr0(cr0);
+}
+
+void
x86_patch(bool early)
{
static bool first, second;
@@ -181,13 +205,7 @@ x86_patch(bool early)
second = true;
}
- /* Disable interrupts. */
- psl = x86_read_psl();
- x86_disable_intr();
-
- /* Disable write protection in supervisor mode. */
- cr0 = rcr0();
- lcr0(cr0 & ~CR0_WP);
+ x86_patch_window_open(&psl, &cr0);
#if !defined(GPROF)
if (!early && ncpu == 1) {
@@ -298,43 +316,5 @@ x86_patch(bool early)
x86_hotpatch(HP_NAME_STAC, stac_bytes, sizeof(stac_bytes));
}
-#ifdef SVS
- if (early && cpu_vendor == CPUVENDOR_INTEL) {
- extern uint8_t svs_enter, svs_enter_end;
- extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
- extern uint8_t svs_leave, svs_leave_end;
- extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
- extern bool svs_enabled;
- uint8_t *bytes;
- size_t size;
-
- svs_enabled = true;
-
- bytes = &svs_enter;
- size = (size_t)&svs_enter_end - (size_t)&svs_enter;
- x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
-
- bytes = &svs_enter_altstack;
- size = (size_t)&svs_enter_altstack_end -
- (size_t)&svs_enter_altstack;
- x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
-
- bytes = &svs_leave;
- size = (size_t)&svs_leave_end - (size_t)&svs_leave;
- x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
-
- bytes = &svs_leave_altstack;
- size = (size_t)&svs_leave_altstack_end -
- (size_t)&svs_leave_altstack;
- x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
- }
-#endif
-
- /* Write back and invalidate cache, flush pipelines. */
- wbinvd();
- x86_flush();
- x86_write_psl(psl);
-
- /* Re-enable write protection. */
- lcr0(cr0);
+ x86_patch_window_close(psl, cr0);
}
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.4 src/sys/arch/x86/x86/svs.c:1.5
--- src/sys/arch/x86/x86/svs.c:1.4 Thu Feb 22 08:56:52 2018
+++ src/sys/arch/x86/x86/svs.c Thu Feb 22 09:41:06 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.4 2018/02/22 08:56:52 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.4 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.5 2018/02/22 09:41:06 maxv Exp $");
#include "opt_svs.h"
@@ -39,7 +39,9 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.4
#include <sys/proc.h>
#include <sys/cpu.h>
+#include <x86/cputypes.h>
#include <machine/cpuvar.h>
+#include <machine/frameasm.h>
#include <uvm/uvm.h>
#include <uvm/uvm_page.h>
@@ -503,12 +505,56 @@ svs_pgg_update(bool enable)
tlbflushg();
}
-void svs_init(void);
+static void
+svs_enable(void)
+{
+ extern uint8_t svs_enter, svs_enter_end;
+ extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
+ extern uint8_t svs_leave, svs_leave_end;
+ extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ svs_enabled = true;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &svs_enter;
+ size = (size_t)&svs_enter_end - (size_t)&svs_enter;
+ x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
+
+ bytes = &svs_enter_altstack;
+ size = (size_t)&svs_enter_altstack_end -
+ (size_t)&svs_enter_altstack;
+ x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
+
+ bytes = &svs_leave;
+ size = (size_t)&svs_leave_end - (size_t)&svs_leave;
+ x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
+
+ bytes = &svs_leave_altstack;
+ size = (size_t)&svs_leave_altstack_end -
+ (size_t)&svs_leave_altstack;
+ x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
void
-svs_init(void)
+svs_init(bool early)
{
- if (svs_enabled)
+ /*
+ * When early, declare that we want to use SVS, and hotpatch the
+ * entry points. When late, remove PG_G from the page tables.
+ */
+ if (early) {
+ if (cpu_vendor != CPUVENDOR_INTEL) {
+ return;
+ }
+ svs_enable();
+ } else if (svs_enabled) {
svs_pgg_update(false);
+ }
}
Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.104 src/sys/arch/x86/x86/x86_machdep.c:1.105
--- src/sys/arch/x86/x86/x86_machdep.c:1.104 Thu Feb 22 08:56:52 2018
+++ src/sys/arch/x86/x86/x86_machdep.c Thu Feb 22 09:41:06 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.104 2018/02/22 08:56:52 maxv Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.104 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.105 2018/02/22 09:41:06 maxv Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -1092,8 +1092,7 @@ void
x86_startup(void)
{
#if SVS
- void svs_init(void);
- svs_init();
+ svs_init(false);
#endif
#if !defined(XEN)
nmi_init();
