Module Name: src
Committed By: maxv
Date: Wed Oct 31 06:26:26 UTC 2018
Modified Files:
src/sys/arch/amd64/amd64: machdep.c
src/sys/arch/amd64/conf: Makefile.amd64 files.amd64
src/sys/arch/amd64/include: Makefile
src/sys/arch/x86/x86: pmap.c
src/sys/kern: files.kern
src/sys/sys: asan.h
Added Files:
src/sys/arch/amd64/include: asan.h
src/sys/kern: subr_asan.c
Removed Files:
src/sys/arch/amd64/amd64: asan.c
Log Message:
Move the MI parts of KASAN into kern/subr_asan.c. This file includes
machine/asan.h, which contains the MD functions. We use an include rather
than a plain C file, because we want GCC to optimize/inline some functions
into one single block.
The amd64 MD parts of KASAN are moved accordingly.
The naming convention we use is:
kasan_*
a generic kasan object, declared in subr_asan.c
kasan_md_*
an MD kasan object, declared in machine/asan.h, and used
in subr_asan.c
__md_*
an MD object, declared in machine/asan.h, and not used
outside
Overall this makes it easier to add KASAN support on more architectures.
Discussed with several people.
To generate a diff of this commit:
cvs rdiff -u -r1.10 -r0 src/sys/arch/amd64/amd64/asan.c
cvs rdiff -u -r1.319 -r1.320 src/sys/arch/amd64/amd64/machdep.c
cvs rdiff -u -r1.74 -r1.75 src/sys/arch/amd64/conf/Makefile.amd64
cvs rdiff -u -r1.106 -r1.107 src/sys/arch/amd64/conf/files.amd64
cvs rdiff -u -r1.20 -r1.21 src/sys/arch/amd64/include/Makefile
cvs rdiff -u -r0 -r1.1 src/sys/arch/amd64/include/asan.h
cvs rdiff -u -r1.308 -r1.309 src/sys/arch/x86/x86/pmap.c
cvs rdiff -u -r1.23 -r1.24 src/sys/kern/files.kern
cvs rdiff -u -r0 -r1.1 src/sys/kern/subr_asan.c
cvs rdiff -u -r1.7 -r1.8 src/sys/sys/asan.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/machdep.c
diff -u src/sys/arch/amd64/amd64/machdep.c:1.319 src/sys/arch/amd64/amd64/machdep.c:1.320
--- src/sys/arch/amd64/amd64/machdep.c:1.319 Sun Sep 23 00:59:59 2018
+++ src/sys/arch/amd64/amd64/machdep.c Wed Oct 31 06:26:25 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.319 2018/09/23 00:59:59 cherry Exp $ */
+/* $NetBSD: machdep.c,v 1.320 2018/10/31 06:26:25 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.319 2018/09/23 00:59:59 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.320 2018/10/31 06:26:25 maxv Exp $");
#include "opt_modular.h"
#include "opt_user_ldt.h"
@@ -152,6 +152,7 @@ __KERNEL_RCSID(0, "$NetBSD: machdep.c,v
#include <sys/device.h>
#include <sys/lwp.h>
#include <sys/proc.h>
+#include <sys/asan.h>
#ifdef KGDB
#include <sys/kgdb.h>
@@ -1683,8 +1684,7 @@ init_x86_64(paddr_t first_avail)
init_pte();
#ifdef KASAN
- void kasan_early_init(void);
- kasan_early_init();
+ kasan_early_init((void *)lwp0uarea);
#endif
uvm_lwp_setuarea(&lwp0, lwp0uarea);
@@ -1766,7 +1766,6 @@ init_x86_64(paddr_t first_avail)
init_x86_msgbuf();
#ifdef KASAN
- void kasan_init(void);
kasan_init();
#endif
Index: src/sys/arch/amd64/conf/Makefile.amd64
diff -u src/sys/arch/amd64/conf/Makefile.amd64:1.74 src/sys/arch/amd64/conf/Makefile.amd64:1.75
--- src/sys/arch/amd64/conf/Makefile.amd64:1.74 Sat Sep 22 12:24:01 2018
+++ src/sys/arch/amd64/conf/Makefile.amd64 Wed Oct 31 06:26:25 2018
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.amd64,v 1.74 2018/09/22 12:24:01 rin Exp $
+# $NetBSD: Makefile.amd64,v 1.75 2018/10/31 06:26:25 maxv Exp $
# Makefile for NetBSD
#
@@ -53,7 +53,7 @@ CFLAGS+= -mindirect-branch-register
KASANFLAGS= -fsanitize=kernel-address \
--param asan-globals=1 --param asan-stack=1 \
-fasan-shadow-offset=0xDFFF900000000000
-.for f in asan.c
+.for f in subr_asan.c
KASANFLAGS.${f}= # empty
.endfor
CFLAGS+= ${KASANFLAGS.${.IMPSRC:T}:U${KASANFLAGS}}
Index: src/sys/arch/amd64/conf/files.amd64
diff -u src/sys/arch/amd64/conf/files.amd64:1.106 src/sys/arch/amd64/conf/files.amd64:1.107
--- src/sys/arch/amd64/conf/files.amd64:1.106 Mon Aug 20 15:04:51 2018
+++ src/sys/arch/amd64/conf/files.amd64 Wed Oct 31 06:26:25 2018
@@ -1,4 +1,4 @@
-# $NetBSD: files.amd64,v 1.106 2018/08/20 15:04:51 maxv Exp $
+# $NetBSD: files.amd64,v 1.107 2018/10/31 06:26:25 maxv Exp $
#
# new style config file for amd64 architecture
#
@@ -39,7 +39,6 @@ file arch/amd64/amd64/spl.S machdep
file arch/amd64/amd64/amd64func.S machdep
file arch/amd64/amd64/amd64_trap.S machdep
-file arch/amd64/amd64/asan.c kasan
file arch/amd64/amd64/autoconf.c machdep
file arch/amd64/amd64/busfunc.S machdep
file arch/amd64/amd64/cpu_in_cksum.S (inet | inet6) & cpu_in_cksum
Index: src/sys/arch/amd64/include/Makefile
diff -u src/sys/arch/amd64/include/Makefile:1.20 src/sys/arch/amd64/include/Makefile:1.21
--- src/sys/arch/amd64/include/Makefile:1.20 Thu Jul 12 10:46:41 2018
+++ src/sys/arch/amd64/include/Makefile Wed Oct 31 06:26:26 2018
@@ -1,8 +1,8 @@
-# $NetBSD: Makefile,v 1.20 2018/07/12 10:46:41 maxv Exp $
+# $NetBSD: Makefile,v 1.21 2018/10/31 06:26:26 maxv Exp $
INCSDIR= /usr/include/amd64
-INCS= ansi.h aout_machdep.h asm.h \
+INCS= ansi.h aout_machdep.h asan.h asm.h \
bootinfo.h bswap.h byte_swap.h \
cdefs.h cpu.h cpu_rng.h\
disklabel.h \
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.308 src/sys/arch/x86/x86/pmap.c:1.309
--- src/sys/arch/x86/x86/pmap.c:1.308 Sat Sep 29 13:19:38 2018
+++ src/sys/arch/x86/x86/pmap.c Wed Oct 31 06:26:26 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.308 2018/09/29 13:19:38 cherry Exp $ */
+/* $NetBSD: pmap.c,v 1.309 2018/10/31 06:26:26 maxv Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.308 2018/09/29 13:19:38 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.309 2018/10/31 06:26:26 maxv Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -149,6 +149,7 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3
#include <sys/intr.h>
#include <sys/xcall.h>
#include <sys/kcore.h>
+#include <sys/asan.h>
#include <uvm/uvm.h>
#include <uvm/pmap/pmap_pvt.h>
@@ -4494,7 +4495,6 @@ pmap_growkernel(vaddr_t maxkvaddr)
#endif
#ifdef KASAN
- void kasan_shadow_map(void *, size_t);
kasan_shadow_map((void *)pmap_maxkvaddr,
(size_t)(maxkvaddr - pmap_maxkvaddr));
#endif
Index: src/sys/kern/files.kern
diff -u src/sys/kern/files.kern:1.23 src/sys/kern/files.kern:1.24
--- src/sys/kern/files.kern:1.23 Fri Sep 14 01:55:19 2018
+++ src/sys/kern/files.kern Wed Oct 31 06:26:26 2018
@@ -1,4 +1,4 @@
-# $NetBSD: files.kern,v 1.23 2018/09/14 01:55:19 mrg Exp $
+# $NetBSD: files.kern,v 1.24 2018/10/31 06:26:26 maxv Exp $
#
# kernel sources
@@ -95,6 +95,7 @@ file kern/kern_uuid.c kern
file kern/kgdb_stub.c kgdb
file kern/sched_4bsd.c sched_4bsd
file kern/sched_m2.c sched_m2
+file kern/subr_asan.c kasan
file kern/subr_autoconf.c kern
file kern/subr_blist.c vmswap
file kern/subr_bufq.c kern
Index: src/sys/sys/asan.h
diff -u src/sys/sys/asan.h:1.7 src/sys/sys/asan.h:1.8
--- src/sys/sys/asan.h:1.7 Wed Aug 22 14:11:26 2018
+++ src/sys/sys/asan.h Wed Oct 31 06:26:26 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: asan.h,v 1.7 2018/08/22 14:11:26 christos Exp $ */
+/* $NetBSD: asan.h,v 1.8 2018/10/31 06:26:26 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -39,6 +39,11 @@
#include <sys/types.h>
#ifdef KASAN
+void kasan_shadow_map(void *, size_t);
+void kasan_early_init(void *);
+void kasan_init(void);
+void kasan_softint(struct lwp *);
+
void kasan_add_redzone(size_t *);
void kasan_alloc(const void *, size_t, size_t);
void kasan_free(const void *, size_t);
Added files:
Index: src/sys/arch/amd64/include/asan.h
diff -u /dev/null src/sys/arch/amd64/include/asan.h:1.1
--- /dev/null Wed Oct 31 06:26:26 2018
+++ src/sys/arch/amd64/include/asan.h Wed Oct 31 06:26:26 2018
@@ -0,0 +1,221 @@
+/* $NetBSD: asan.h,v 1.1 2018/10/31 06:26:26 maxv Exp $ */
+
+/*
+ * Copyright (c) 2018 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/ksyms.h>
+
+#include <amd64/pmap.h>
+#include <amd64/vmparam.h>
+
+#ifdef __HAVE_PCPU_AREA
+#error "PCPU area not allowed with KASAN"
+#endif
+#ifdef __HAVE_DIRECT_MAP
+#error "DMAP not allowed with KASAN"
+#endif
+
+#define __MD_VIRTUAL_SHIFT 47 /* 48bit address space, cut half */
+#define __MD_CANONICAL_BASE 0xFFFF800000000000
+
+#define __MD_SHADOW_SIZE (1ULL << (__MD_VIRTUAL_SHIFT - KASAN_SHADOW_SCALE_SHIFT))
+#define KASAN_MD_SHADOW_START (VA_SIGN_NEG((L4_SLOT_KASAN * NBPD_L4)))
+#define KASAN_MD_SHADOW_END (KASAN_MD_SHADOW_START + __MD_SHADOW_SIZE)
+
+static bool __md_early __read_mostly = true;
+static uint8_t __md_earlypages[8 * PAGE_SIZE] __aligned(PAGE_SIZE);
+static size_t __md_earlytaken = 0;
+
+static inline int8_t *
+kasan_md_addr_to_shad(const void *addr)
+{
+ vaddr_t va = (vaddr_t)addr;
+ return (int8_t *)(KASAN_MD_SHADOW_START +
+ ((va - __MD_CANONICAL_BASE) >> KASAN_SHADOW_SCALE_SHIFT));
+}
+
+static inline bool
+kasan_md_unsupported(vaddr_t addr)
+{
+ return (addr >= (vaddr_t)PTE_BASE &&
+ addr < ((vaddr_t)PTE_BASE + NBPD_L4));
+}
+
+static paddr_t
+__md_early_palloc(void)
+{
+ paddr_t ret;
+
+ KASSERT(__md_earlytaken < 8);
+
+ ret = (paddr_t)(&__md_earlypages[0] + __md_earlytaken * PAGE_SIZE);
+ __md_earlytaken++;
+
+ ret -= KERNBASE;
+
+ return ret;
+}
+
+static paddr_t
+__md_palloc(void)
+{
+ paddr_t pa;
+
+ if (__predict_false(__md_early))
+ pa = __md_early_palloc();
+ else
+ pa = pmap_get_physpage();
+
+ return pa;
+}
+
+static void
+kasan_md_shadow_map_page(vaddr_t va)
+{
+ paddr_t pa;
+
+ if (!pmap_valid_entry(L4_BASE[pl4_i(va)])) {
+ pa = __md_palloc();
+ L4_BASE[pl4_i(va)] = pa | PG_KW | pmap_pg_nx | PG_V;
+ }
+ if (!pmap_valid_entry(L3_BASE[pl3_i(va)])) {
+ pa = __md_palloc();
+ L3_BASE[pl3_i(va)] = pa | PG_KW | pmap_pg_nx | PG_V;
+ }
+ if (!pmap_valid_entry(L2_BASE[pl2_i(va)])) {
+ pa = __md_palloc();
+ L2_BASE[pl2_i(va)] = pa | PG_KW | pmap_pg_nx | PG_V;
+ }
+ if (!pmap_valid_entry(L1_BASE[pl1_i(va)])) {
+ pa = __md_palloc();
+ L1_BASE[pl1_i(va)] = pa | PG_KW | pmap_pg_g | pmap_pg_nx | PG_V;
+ }
+}
+
+/*
+ * Map only the current stack. We will map the rest in kasan_init.
+ */
+static void
+kasan_md_early_init(void *stack)
+{
+ kasan_shadow_map(stack, USPACE);
+ __md_early = false;
+}
+
+/*
+ * Create the shadow mapping. We don't create the 'User' area, because we
+ * exclude it from the monitoring. The 'Main' area is created dynamically
+ * in pmap_growkernel.
+ */
+static void
+kasan_md_init(void)
+{
+ extern struct bootspace bootspace;
+ size_t i;
+
+ CTASSERT((__MD_SHADOW_SIZE / NBPD_L4) == NL4_SLOT_KASAN);
+
+ /* Kernel. */
+ for (i = 0; i < BTSPACE_NSEGS; i++) {
+ if (bootspace.segs[i].type == BTSEG_NONE) {
+ continue;
+ }
+ kasan_shadow_map((void *)bootspace.segs[i].va,
+ bootspace.segs[i].sz);
+ }
+
+ /* Boot region. */
+ kasan_shadow_map((void *)bootspace.boot.va, bootspace.boot.sz);
+
+ /* Module map. */
+ kasan_shadow_map((void *)bootspace.smodule,
+ (size_t)(bootspace.emodule - bootspace.smodule));
+
+ /* The bootstrap spare va. */
+ kasan_shadow_map((void *)bootspace.spareva, PAGE_SIZE);
+}
+
+static inline bool
+__md_unwind_end(const char *name)
+{
+ if (!strcmp(name, "syscall") ||
+ !strcmp(name, "handle_syscall") ||
+ !strncmp(name, "Xintr", 5) ||
+ !strncmp(name, "Xhandle", 7) ||
+ !strncmp(name, "Xresume", 7) ||
+ !strncmp(name, "Xstray", 6) ||
+ !strncmp(name, "Xhold", 5) ||
+ !strncmp(name, "Xrecurse", 8) ||
+ !strcmp(name, "Xdoreti") ||
+ !strncmp(name, "Xsoft", 5)) {
+ return true;
+ }
+
+ return false;
+}
+
+static void
+kasan_md_unwind(void)
+{
+ uint64_t *rbp, rip;
+ const char *mod;
+ const char *sym;
+ size_t nsym;
+ int error;
+
+ rbp = (uint64_t *)__builtin_frame_address(0);
+ nsym = 0;
+
+ while (1) {
+ /* 8(%rbp) contains the saved %rip. */
+ rip = *(rbp + 1);
+
+ if (rip < KERNBASE) {
+ break;
+ }
+ error = ksyms_getname(&mod, &sym, (vaddr_t)rip, KSYMS_PROC);
+ if (error) {
+ break;
+ }
+ printf("#%zu %p in %s <%s>\n", nsym, (void *)rip, sym, mod);
+ if (__md_unwind_end(sym)) {
+ break;
+ }
+
+ rbp = (uint64_t *)*(rbp);
+ if (rbp == 0) {
+ break;
+ }
+ nsym++;
+
+ if (nsym >= 15) {
+ break;
+ }
+ }
+}
Index: src/sys/kern/subr_asan.c
diff -u /dev/null src/sys/kern/subr_asan.c:1.1
--- /dev/null Wed Oct 31 06:26:26 2018
+++ src/sys/kern/subr_asan.c Wed Oct 31 06:26:26 2018
@@ -0,0 +1,536 @@
+/* $NetBSD: subr_asan.c,v 1.1 2018/10/31 06:26:26 maxv Exp $ */
+
+/*
+ * Copyright (c) 2018 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.1 2018/10/31 06:26:26 maxv Exp $");
+
+#include <sys/param.h>
+#include <sys/device.h>
+#include <sys/kernel.h>
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/asan.h>
+
+#include <uvm/uvm.h>
+
+/* ASAN constants. Part of the compiler ABI. */
+#define KASAN_SHADOW_SCALE_SHIFT 3
+#define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
+#define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1)
+
+/* The MD code. */
+#include <machine/asan.h>
+
+/* Our redzone values. */
+#define KASAN_GLOBAL_REDZONE 0xFA
+#define KASAN_MEMORY_REDZONE 0xFB
+
+/* Stack redzone values. Part of the compiler ABI. */
+#define KASAN_STACK_LEFT 0xF1
+#define KASAN_STACK_MID 0xF2
+#define KASAN_STACK_RIGHT 0xF3
+#define KASAN_STACK_PARTIAL 0xF4
+#define KASAN_USE_AFTER_SCOPE 0xF8
+
+/* ASAN ABI version. */
+#if defined(__clang__) && (__clang_major__ - 0 >= 6)
+#define ASAN_ABI_VERSION 8
+#elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
+#define ASAN_ABI_VERSION 8
+#elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
+#define ASAN_ABI_VERSION 6
+#else
+#error "Unsupported compiler version"
+#endif
+
+#define __RET_ADDR (unsigned long)__builtin_return_address(0)
+
+/* Global variable descriptor. Part of the compiler ABI. */
+struct __asan_global_source_location {
+ const char *filename;
+ int line_no;
+ int column_no;
+};
+struct __asan_global {
+ const void *beg; /* address of the global variable */
+ size_t size; /* size of the global variable */
+ size_t size_with_redzone; /* size with the redzone */
+ const void *name; /* name of the variable */
+ const void *module_name; /* name of the module where the var is declared */
+ unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
+ struct __asan_global_source_location *location;
+#if ASAN_ABI_VERSION >= 7
+ uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
+#endif
+};
+
+static bool kasan_enabled __read_mostly = false;
+
+/* -------------------------------------------------------------------------- */
+
+void
+kasan_shadow_map(void *addr, size_t size)
+{
+ size_t sz, npages, i;
+ vaddr_t sva, eva;
+
+ KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
+
+ sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE;
+
+ sva = (vaddr_t)kasan_md_addr_to_shad(addr);
+ eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz;
+
+ sva = rounddown(sva, PAGE_SIZE);
+ eva = roundup(eva, PAGE_SIZE);
+
+ npages = (eva - sva) / PAGE_SIZE;
+
+ KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END);
+
+ for (i = 0; i < npages; i++) {
+ kasan_md_shadow_map_page(sva + i * PAGE_SIZE);
+ }
+}
+
+static void
+kasan_ctors(void)
+{
+ extern uint64_t __CTOR_LIST__, __CTOR_END__;
+ size_t nentries, i;
+ uint64_t *ptr;
+
+ nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) /
+ sizeof(uintptr_t);
+
+ ptr = &__CTOR_LIST__;
+ for (i = 0; i < nentries; i++) {
+ void (*func)(void);
+
+ func = (void *)(*ptr);
+ (*func)();
+
+ ptr++;
+ }
+}
+
+void
+kasan_early_init(void *stack)
+{
+ kasan_md_early_init(stack);
+}
+
+void
+kasan_init(void)
+{
+ /* MD initialization. */
+ kasan_md_init();
+
+ /* Now officially enabled. */
+ kasan_enabled = true;
+
+ /* Call the ASAN constructors. */
+ kasan_ctors();
+}
+
+static void
+kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc)
+{
+ printf("kASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s]\n",
+ (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""),
+ (write ? "write" : "read"));
+ kasan_md_unwind();
+}
+
+static __always_inline void
+kasan_shadow_1byte_markvalid(unsigned long addr)
+{
+ int8_t *byte = kasan_md_addr_to_shad((void *)addr);
+ int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
+
+ *byte = last;
+}
+
+static __always_inline void
+kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t val)
+{
+ void *shad;
+
+ if (__predict_false(size == 0))
+ return;
+ if (__predict_false(kasan_md_unsupported((vaddr_t)addr)))
+ return;
+
+ KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
+ KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
+
+ shad = (void *)kasan_md_addr_to_shad(addr);
+ size = size >> KASAN_SHADOW_SCALE_SHIFT;
+
+ __builtin_memset(shad, val, size);
+}
+
+void
+kasan_add_redzone(size_t *size)
+{
+ *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE);
+ *size += KASAN_SHADOW_SCALE_SIZE;
+}
+
+static void
+kasan_markmem(const void *addr, size_t size, bool valid)
+{
+ size_t i;
+
+ KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
+
+ if (valid) {
+ for (i = 0; i < size; i++) {
+ kasan_shadow_1byte_markvalid((unsigned long)addr+i);
+ }
+ } else {
+ KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
+ kasan_shadow_Nbyte_fill(addr, size, KASAN_MEMORY_REDZONE);
+ }
+}
+
+void
+kasan_softint(struct lwp *l)
+{
+ const void *stk = (const void *)uvm_lwp_getuarea(l);
+
+ kasan_shadow_Nbyte_fill(stk, USPACE, 0);
+}
+
+void
+kasan_alloc(const void *addr, size_t size, size_t sz_with_redz)
+{
+ kasan_markmem(addr, sz_with_redz, false);
+ kasan_markmem(addr, size, true);
+}
+
+void
+kasan_free(const void *addr, size_t sz_with_redz)
+{
+ kasan_markmem(addr, sz_with_redz, true);
+}
+
+/* -------------------------------------------------------------------------- */
+
+#define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
+ (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
+ ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
+
+static __always_inline bool
+kasan_shadow_1byte_isvalid(unsigned long addr)
+{
+ int8_t *byte = kasan_md_addr_to_shad((void *)addr);
+ int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
+
+ return __predict_true(*byte == 0 || last <= *byte);
+}
+
+static __always_inline bool
+kasan_shadow_2byte_isvalid(unsigned long addr)
+{
+ int8_t *byte, last;
+
+ if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
+ return (kasan_shadow_1byte_isvalid(addr) &&
+ kasan_shadow_1byte_isvalid(addr+1));
+ }
+
+ byte = kasan_md_addr_to_shad((void *)addr);
+ last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
+
+ return __predict_true(*byte == 0 || last <= *byte);
+}
+
+static __always_inline bool
+kasan_shadow_4byte_isvalid(unsigned long addr)
+{
+ int8_t *byte, last;
+
+ if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
+ return (kasan_shadow_2byte_isvalid(addr) &&
+ kasan_shadow_2byte_isvalid(addr+2));
+ }
+
+ byte = kasan_md_addr_to_shad((void *)addr);
+ last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
+
+ return __predict_true(*byte == 0 || last <= *byte);
+}
+
+static __always_inline bool
+kasan_shadow_8byte_isvalid(unsigned long addr)
+{
+ int8_t *byte, last;
+
+ if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
+ return (kasan_shadow_4byte_isvalid(addr) &&
+ kasan_shadow_4byte_isvalid(addr+4));
+ }
+
+ byte = kasan_md_addr_to_shad((void *)addr);
+ last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
+
+ return __predict_true(*byte == 0 || last <= *byte);
+}
+
+static __always_inline bool
+kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size)
+{
+ size_t i;
+
+ for (i = 0; i < size; i++) {
+ if (!kasan_shadow_1byte_isvalid(addr+i))
+ return false;
+ }
+
+ return true;
+}
+
+static __always_inline void
+kasan_shadow_check(unsigned long addr, size_t size, bool write,
+ unsigned long retaddr)
+{
+ bool valid;
+
+ if (__predict_false(!kasan_enabled))
+ return;
+ if (__predict_false(size == 0))
+ return;
+ if (__predict_false(kasan_md_unsupported(addr)))
+ return;
+
+ if (__builtin_constant_p(size)) {
+ switch (size) {
+ case 1:
+ valid = kasan_shadow_1byte_isvalid(addr);
+ break;
+ case 2:
+ valid = kasan_shadow_2byte_isvalid(addr);
+ break;
+ case 4:
+ valid = kasan_shadow_4byte_isvalid(addr);
+ break;
+ case 8:
+ valid = kasan_shadow_8byte_isvalid(addr);
+ break;
+ default:
+ valid = kasan_shadow_Nbyte_isvalid(addr, size);
+ break;
+ }
+ } else {
+ valid = kasan_shadow_Nbyte_isvalid(addr, size);
+ }
+
+ if (__predict_false(!valid)) {
+ kasan_report(addr, size, write, retaddr);
+ }
+}
+
+/* -------------------------------------------------------------------------- */
+
+void *
+kasan_memcpy(void *dst, const void *src, size_t len)
+{
+ kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
+ kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
+ return __builtin_memcpy(dst, src, len);
+}
+
+int
+kasan_memcmp(const void *b1, const void *b2, size_t len)
+{
+ kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
+ kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
+ return __builtin_memcmp(b1, b2, len);
+}
+
+void *
+kasan_memset(void *b, int c, size_t len)
+{
+ kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
+ return __builtin_memset(b, c, len);
+}
+
+char *
+kasan_strcpy(char *dst, const char *src)
+{
+ char *save = dst;
+
+ while (1) {
+ kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
+ kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
+ *dst = *src;
+ if (*src == '\0')
+ break;
+ src++, dst++;
+ }
+
+ return save;
+}
+
+int
+kasan_strcmp(const char *s1, const char *s2)
+{
+ while (1) {
+ kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
+ kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
+ if (*s1 != *s2)
+ break;
+ if (*s1 == '\0')
+ return 0;
+ s1++, s2++;
+ }
+
+ return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
+}
+
+size_t
+kasan_strlen(const char *str)
+{
+ const char *s;
+
+ s = str;
+ while (1) {
+ kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
+ if (*s == '\0')
+ break;
+ s++;
+ }
+
+ return (s - str);
+}
+
+/* -------------------------------------------------------------------------- */
+
+void __asan_register_globals(struct __asan_global *, size_t);
+void __asan_unregister_globals(struct __asan_global *, size_t);
+
+void
+__asan_register_globals(struct __asan_global *globals, size_t n)
+{
+ size_t i;
+
+ for (i = 0; i < n; i++) {
+ kasan_alloc(globals[i].beg, globals[i].size,
+ globals[i].size_with_redzone);
+ }
+}
+
+void
+__asan_unregister_globals(struct __asan_global *globals, size_t n)
+{
+ /* never called */
+}
+
+#define ASAN_LOAD_STORE(size) \
+ void __asan_load##size(unsigned long); \
+ void __asan_load##size(unsigned long addr) \
+ { \
+ kasan_shadow_check(addr, size, false, __RET_ADDR);\
+ } \
+ void __asan_load##size##_noabort(unsigned long); \
+ void __asan_load##size##_noabort(unsigned long addr) \
+ { \
+ kasan_shadow_check(addr, size, false, __RET_ADDR);\
+ } \
+ void __asan_store##size(unsigned long); \
+ void __asan_store##size(unsigned long addr) \
+ { \
+ kasan_shadow_check(addr, size, true, __RET_ADDR);\
+ } \
+ void __asan_store##size##_noabort(unsigned long); \
+ void __asan_store##size##_noabort(unsigned long addr) \
+ { \
+ kasan_shadow_check(addr, size, true, __RET_ADDR);\
+ }
+
+ASAN_LOAD_STORE(1);
+ASAN_LOAD_STORE(2);
+ASAN_LOAD_STORE(4);
+ASAN_LOAD_STORE(8);
+ASAN_LOAD_STORE(16);
+
+void __asan_loadN(unsigned long, size_t);
+void __asan_loadN_noabort(unsigned long, size_t);
+void __asan_storeN(unsigned long, size_t);
+void __asan_storeN_noabort(unsigned long, size_t);
+void __asan_handle_no_return(void);
+
+void
+__asan_loadN(unsigned long addr, size_t size)
+{
+ kasan_shadow_check(addr, size, false, __RET_ADDR);
+}
+
+void
+__asan_loadN_noabort(unsigned long addr, size_t size)
+{
+ kasan_shadow_check(addr, size, false, __RET_ADDR);
+}
+
+void
+__asan_storeN(unsigned long addr, size_t size)
+{
+ kasan_shadow_check(addr, size, true, __RET_ADDR);
+}
+
+void
+__asan_storeN_noabort(unsigned long addr, size_t size)
+{
+ kasan_shadow_check(addr, size, true, __RET_ADDR);
+}
+
+void
+__asan_handle_no_return(void)
+{
+ /* nothing */
+}
+
+#define ASAN_SET_SHADOW(byte) \
+ void __asan_set_shadow_##byte(void *, size_t); \
+ void __asan_set_shadow_##byte(void *addr, size_t size) \
+ { \
+ __builtin_memset((void *)addr, 0x##byte, size); \
+ }
+
+ASAN_SET_SHADOW(00);
+ASAN_SET_SHADOW(f1);
+ASAN_SET_SHADOW(f2);
+ASAN_SET_SHADOW(f3);
+ASAN_SET_SHADOW(f5);
+ASAN_SET_SHADOW(f8);
