Module Name: src
Committed By: maxv
Date: Wed Apr 15 16:28:29 UTC 2020
Modified Files:
src/sys/arch/amd64/include: msan.h
src/sys/kern: subr_msan.c
Log Message:
Use large pages for the kMSan shadows. This greatly improves performance,
and slightly reduces memory consumption.
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/amd64/include/msan.h
cvs rdiff -u -r1.9 -r1.10 src/sys/kern/subr_msan.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/include/msan.h
diff -u src/sys/arch/amd64/include/msan.h:1.1 src/sys/arch/amd64/include/msan.h:1.2
--- src/sys/arch/amd64/include/msan.h:1.1 Thu Nov 14 16:23:52 2019
+++ src/sys/arch/amd64/include/msan.h Wed Apr 15 16:28:29 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: msan.h,v 1.1 2019/11/14 16:23:52 maxv Exp $ */
+/* $NetBSD: msan.h,v 1.2 2020/04/15 16:28:29 maxv Exp $ */
/*
* Copyright (c) 2019 The NetBSD Foundation, Inc.
@@ -97,31 +97,62 @@ kmsan_md_unsupported(vaddr_t addr)
static inline paddr_t
__md_palloc(void)
{
+ /* The page is zeroed. */
return pmap_get_physpage();
}
+static inline paddr_t
+__md_palloc_large(void)
+{
+ struct pglist pglist;
+ int ret;
+
+ if (!uvm.page_init_done)
+ return 0;
+
+ kmsan_init_arg(sizeof(psize_t) + 4 * sizeof(paddr_t) +
+ sizeof(struct pglist *) + 2 * sizeof(int));
+ ret = uvm_pglistalloc(NBPD_L2, 0, ~0UL, NBPD_L2, 0,
+ &pglist, 1, 0);
+ if (ret != 0)
+ return 0;
+
+ /* The page may not be zeroed. */
+ return VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
+}
+
static void
kmsan_md_shadow_map_page(vaddr_t va)
{
+ const pt_entry_t pteflags = PTE_W | pmap_pg_nx | PTE_P;
paddr_t pa;
KASSERT(va >= __MD_SHADOW_START && va < __MD_SHADOW_END);
if (!pmap_valid_entry(L4_BASE[pl4_i(va)])) {
pa = __md_palloc();
- L4_BASE[pl4_i(va)] = pa | PTE_W | pmap_pg_nx | PTE_P;
+ L4_BASE[pl4_i(va)] = pa | pteflags;
}
if (!pmap_valid_entry(L3_BASE[pl3_i(va)])) {
pa = __md_palloc();
- L3_BASE[pl3_i(va)] = pa | PTE_W | pmap_pg_nx | PTE_P;
+ L3_BASE[pl3_i(va)] = pa | pteflags;
}
if (!pmap_valid_entry(L2_BASE[pl2_i(va)])) {
+ if ((pa = __md_palloc_large()) != 0) {
+ L2_BASE[pl2_i(va)] = pa | pteflags | PTE_PS |
+ pmap_pg_g;
+ __insn_barrier();
+ __builtin_memset(va, 0, NBPD_L2);
+ return;
+ }
pa = __md_palloc();
- L2_BASE[pl2_i(va)] = pa | PTE_W | pmap_pg_nx | PTE_P;
+ L2_BASE[pl2_i(va)] = pa | pteflags;
+ } else if (L2_BASE[pl2_i(va)] & PTE_PS) {
+ return;
}
if (!pmap_valid_entry(L1_BASE[pl1_i(va)])) {
pa = __md_palloc();
- L1_BASE[pl1_i(va)] = pa | PTE_W | pmap_pg_g | pmap_pg_nx | PTE_P;
+ L1_BASE[pl1_i(va)] = pa | pteflags | pmap_pg_g;
}
}
Index: src/sys/kern/subr_msan.c
diff -u src/sys/kern/subr_msan.c:1.9 src/sys/kern/subr_msan.c:1.10
--- src/sys/kern/subr_msan.c:1.9 Fri Apr 3 18:26:14 2020
+++ src/sys/kern/subr_msan.c Wed Apr 15 16:28:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_msan.c,v 1.9 2020/04/03 18:26:14 maxv Exp $ */
+/* $NetBSD: subr_msan.c,v 1.10 2020/04/15 16:28:28 maxv Exp $ */
/*
* Copyright (c) 2019-2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.9 2020/04/03 18:26:14 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.10 2020/04/15 16:28:28 maxv Exp $");
#include <sys/param.h>
#include <sys/device.h>
@@ -50,6 +50,9 @@ __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,
static void kmsan_printf(const char *, ...);
+void kmsan_init_arg(size_t);
+void kmsan_init_ret(size_t);
+
#ifdef KMSAN_PANIC
#define REPORT panic
#else
@@ -356,9 +359,6 @@ kmsan_shadow_check(const void *addr, siz
}
}
-void kmsan_init_arg(size_t);
-void kmsan_init_ret(size_t);
-
void
kmsan_init_arg(size_t n)
{
