Module Name: src
Committed By: maxv
Date: Tue Nov 5 20:19:18 UTC 2019
Modified Files:
src/share/mk: bsd.sys.mk
src/sys/arch/amd64/amd64: machdep.c mptramp.S
src/sys/arch/amd64/conf: GENERIC Makefile.amd64
src/sys/arch/x86/x86: cpu.c
src/sys/conf: files
src/sys/kern: files.kern
src/sys/lib/libkern: libkern.h
src/sys/sys: atomic.h bus_proto.h cdefs.h systm.h
Added Files:
src/sys/arch/amd64/include: csan.h
src/sys/kern: subr_csan.c
src/sys/sys: csan.h
Log Message:
Add Kernel Concurrency Sanitizer (kCSan) support. This sanitizer allows us
to detect race conditions at runtime. It is a variation of TSan that is
easy to implement and more suited to kernel internals, albeit theoretically
less precise than TSan's happens-before.
We do basically two things:
- On every KCSAN_NACCESSES (=2000) memory accesses, we create a cell
describing the access, and delay the calling CPU (10ms).
- On all memory accesses, we verify if the memory we're reading/writing
is referenced in a cell already.
The combination of the two means that, if for example cpu0 does a read that
is selected and cpu1 does a write at the same address, kCSan will fire,
because cpu1's write collides with cpu0's read cell.
The coverage of the instrumentation is the same as that of kASan. Also, the
code is organized in a way similar to kASan, so it is easy to add support
for more architectures than amd64. kCSan is compatible with KCOV.
Reviewed by Kamil.
To generate a diff of this commit:
cvs rdiff -u -r1.294 -r1.295 src/share/mk/bsd.sys.mk
cvs rdiff -u -r1.337 -r1.338 src/sys/arch/amd64/amd64/machdep.c
cvs rdiff -u -r1.26 -r1.27 src/sys/arch/amd64/amd64/mptramp.S
cvs rdiff -u -r1.544 -r1.545 src/sys/arch/amd64/conf/GENERIC
cvs rdiff -u -r1.78 -r1.79 src/sys/arch/amd64/conf/Makefile.amd64
cvs rdiff -u -r0 -r1.1 src/sys/arch/amd64/include/csan.h
cvs rdiff -u -r1.173 -r1.174 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.1240 -r1.1241 src/sys/conf/files
cvs rdiff -u -r1.35 -r1.36 src/sys/kern/files.kern
cvs rdiff -u -r0 -r1.1 src/sys/kern/subr_csan.c
cvs rdiff -u -r1.132 -r1.133 src/sys/lib/libkern/libkern.h
cvs rdiff -u -r1.15 -r1.16 src/sys/sys/atomic.h
cvs rdiff -u -r1.9 -r1.10 src/sys/sys/bus_proto.h
cvs rdiff -u -r1.147 -r1.148 src/sys/sys/cdefs.h
cvs rdiff -u -r0 -r1.1 src/sys/sys/csan.h
cvs rdiff -u -r1.287 -r1.288 src/sys/sys/systm.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/share/mk/bsd.sys.mk
diff -u src/share/mk/bsd.sys.mk:1.294 src/share/mk/bsd.sys.mk:1.295
--- src/share/mk/bsd.sys.mk:1.294 Tue Oct 15 15:05:00 2019
+++ src/share/mk/bsd.sys.mk Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.sys.mk,v 1.294 2019/10/15 15:05:00 christos Exp $
+# $NetBSD: bsd.sys.mk,v 1.295 2019/11/05 20:19:17 maxv Exp $
#
# Build definitions used for NetBSD source tree builds.
@@ -246,7 +246,8 @@ CFLAGS+= ${KLEAKFLAGS.${.IMPSRC:T}:U${KL
.if ${KCOV:U0} > 0
KCOVFLAGS= -fsanitize-coverage=trace-pc
-.for f in subr_kcov.c subr_lwp_specificdata.c subr_specificdata.c subr_asan.c
+.for f in subr_kcov.c subr_lwp_specificdata.c subr_specificdata.c subr_asan.c \
+ subr_csan.c
KCOVFLAGS.${f}= # empty
.endfor
CFLAGS+= ${KCOVFLAGS.${.IMPSRC:T}:U${KCOVFLAGS}}
Index: src/sys/arch/amd64/amd64/machdep.c
diff -u src/sys/arch/amd64/amd64/machdep.c:1.337 src/sys/arch/amd64/amd64/machdep.c:1.338
--- src/sys/arch/amd64/amd64/machdep.c:1.337 Sat Oct 12 06:31:03 2019
+++ src/sys/arch/amd64/amd64/machdep.c Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.338 2019/11/05 20:19:17 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.338 2019/11/05 20:19:17 maxv Exp $");
#include "opt_modular.h"
#include "opt_user_ldt.h"
@@ -152,6 +152,7 @@ __KERNEL_RCSID(0, "$NetBSD: machdep.c,v
#include <sys/lwp.h>
#include <sys/proc.h>
#include <sys/asan.h>
+#include <sys/csan.h>
#ifdef KGDB
#include <sys/kgdb.h>
@@ -1761,6 +1762,7 @@ init_x86_64(paddr_t first_avail)
#ifdef KASAN
kasan_init();
#endif
+ kcsan_init();
pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 32 * 1024 * 1024);
Index: src/sys/arch/amd64/amd64/mptramp.S
diff -u src/sys/arch/amd64/amd64/mptramp.S:1.26 src/sys/arch/amd64/amd64/mptramp.S:1.27
--- src/sys/arch/amd64/amd64/mptramp.S:1.26 Sun Nov 26 14:54:43 2017
+++ src/sys/arch/amd64/amd64/mptramp.S Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: mptramp.S,v 1.26 2017/11/26 14:54:43 maxv Exp $ */
+/* $NetBSD: mptramp.S,v 1.27 2019/11/05 20:19:17 maxv Exp $ */
/*
* Copyright (c) 2000, 2016 The NetBSD Foundation, Inc.
@@ -75,6 +75,7 @@
*/
#include "assym.h"
+#include "opt_kcsan.h"
#include <machine/asm.h>
#include <machine/specialreg.h>
#include <machine/segments.h>
@@ -244,6 +245,18 @@ _C_LABEL(cpu_spinup_trampoline_end): /*
movl PCB_CR0(%rsi),%eax
movq %rax,%cr0
+#ifdef KCSAN
+ /*
+ * The C instrumentation uses GS.base, so initialize it right now. It
+ * gets re-initialized later, that's fine.
+ */
+ movl $MSR_GSBASE,%ecx
+ movq %rdi,%rax
+ movq %rdi,%rdx
+ shrq $32,%rdx
+ wrmsr
+#endif
+
call _C_LABEL(cpu_hatch)
END(cpu_spinup_trampoline)
Index: src/sys/arch/amd64/conf/GENERIC
diff -u src/sys/arch/amd64/conf/GENERIC:1.544 src/sys/arch/amd64/conf/GENERIC:1.545
--- src/sys/arch/amd64/conf/GENERIC:1.544 Fri Nov 1 02:53:23 2019
+++ src/sys/arch/amd64/conf/GENERIC Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-# $NetBSD: GENERIC,v 1.544 2019/11/01 02:53:23 msaitoh Exp $
+# $NetBSD: GENERIC,v 1.545 2019/11/05 20:19:17 maxv Exp $
#
# GENERIC machine description file
#
@@ -22,7 +22,7 @@ include "arch/amd64/conf/std.amd64"
options INCLUDE_CONFIG_FILE # embed config file in kernel binary
-#ident "GENERIC-$Revision: 1.544 $"
+#ident "GENERIC-$Revision: 1.545 $"
maxusers 64 # estimated number of users
@@ -128,6 +128,11 @@ options KDTRACE_HOOKS # kernel DTrace h
#options POOL_QUARANTINE # optional
#options KASAN_PANIC # optional
+# Kernel Concurrency Sanitizer (kCSan).
+#makeoptions KCSAN=1 # mandatory
+#options KCSAN # mandatory
+#options KCSAN_PANIC # optional
+
# Kernel Info Leak Detector.
#makeoptions KLEAK=1
#options KLEAK
Index: src/sys/arch/amd64/conf/Makefile.amd64
diff -u src/sys/arch/amd64/conf/Makefile.amd64:1.78 src/sys/arch/amd64/conf/Makefile.amd64:1.79
--- src/sys/arch/amd64/conf/Makefile.amd64:1.78 Sat Sep 7 18:56:01 2019
+++ src/sys/arch/amd64/conf/Makefile.amd64 Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.amd64,v 1.78 2019/09/07 18:56:01 maxv Exp $
+# $NetBSD: Makefile.amd64,v 1.79 2019/11/05 20:19:17 maxv Exp $
# Makefile for NetBSD
#
@@ -60,6 +60,15 @@ KASANFLAGS.${f}= # empty
CFLAGS+= ${KASANFLAGS.${.IMPSRC:T}:U${KASANFLAGS}}
.endif
+.if ${KCSAN:U0} > 0 && ${HAVE_GCC:U0} > 0
+KCSANFLAGS= -fsanitize=thread
+.for f in subr_csan.c clock.c lapic.c subr_kcov.c subr_lwp_specificdata.c \
+ subr_specificdata.c
+KCSANFLAGS.${f}= # empty
+.endfor
+CFLAGS+= ${KCSANFLAGS.${.IMPSRC:T}:U${KCSANFLAGS}}
+.endif
+
##
## (3) libkern and compat
##
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.173 src/sys/arch/x86/x86/cpu.c:1.174
--- src/sys/arch/x86/x86/cpu.c:1.173 Sat Oct 12 06:31:04 2019
+++ src/sys/arch/x86/x86/cpu.c Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.173 2019/10/12 06:31:04 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.174 2019/11/05 20:19:17 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.173 2019/10/12 06:31:04 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.174 2019/11/05 20:19:17 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -82,6 +82,7 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.17
#include <sys/idle.h>
#include <sys/atomic.h>
#include <sys/reboot.h>
+#include <sys/csan.h>
#include <uvm/uvm.h>
@@ -429,6 +430,7 @@ cpu_attach(device_t parent, device_t sel
#endif
/* Make sure DELAY() is initialized. */
DELAY(1);
+ kcsan_cpu_init(ci);
again = true;
}
@@ -961,6 +963,8 @@ cpu_hatch(void *v)
aprint_debug_dev(ci->ci_dev, "running\n");
+ kcsan_cpu_init(ci);
+
idle_loop(NULL);
KASSERT(false);
}
Index: src/sys/conf/files
diff -u src/sys/conf/files:1.1240 src/sys/conf/files:1.1241
--- src/sys/conf/files:1.1240 Sun Sep 22 22:59:38 2019
+++ src/sys/conf/files Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-# $NetBSD: files,v 1.1240 2019/09/22 22:59:38 christos Exp $
+# $NetBSD: files,v 1.1241 2019/11/05 20:19:17 maxv Exp $
# @(#)files.newconf 7.5 (Berkeley) 5/10/93
version 20171118
@@ -31,6 +31,8 @@ defflag opt_diagnostic.h _DIAGNOSTIC
defflag GPROF
defflag KASAN
defflag opt_kasan.h KASAN_PANIC
+defflag KCSAN
+defflag opt_kcsan.h KCSAN_PANIC
defflag KLEAK
defflag KCOV
defflag opt_pool.h POOL_QUARANTINE
Index: src/sys/kern/files.kern
diff -u src/sys/kern/files.kern:1.35 src/sys/kern/files.kern:1.36
--- src/sys/kern/files.kern:1.35 Thu Aug 15 12:24:08 2019
+++ src/sys/kern/files.kern Tue Nov 5 20:19:17 2019
@@ -1,4 +1,4 @@
-# $NetBSD: files.kern,v 1.35 2019/08/15 12:24:08 maxv Exp $
+# $NetBSD: files.kern,v 1.36 2019/11/05 20:19:17 maxv Exp $
#
# kernel sources
@@ -105,6 +105,7 @@ file kern/subr_callback.c kern
file kern/subr_cprng.c kern
file kern/subr_cpufreq.c kern
file kern/subr_copy.c kern
+file kern/subr_csan.c kcsan
file kern/subr_debug.c debug
file kern/subr_device.c kern
file kern/subr_devsw.c kern
Index: src/sys/lib/libkern/libkern.h
diff -u src/sys/lib/libkern/libkern.h:1.132 src/sys/lib/libkern/libkern.h:1.133
--- src/sys/lib/libkern/libkern.h:1.132 Fri Sep 20 13:38:00 2019
+++ src/sys/lib/libkern/libkern.h Tue Nov 5 20:19:18 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: libkern.h,v 1.132 2019/09/20 13:38:00 maxv Exp $ */
+/* $NetBSD: libkern.h,v 1.133 2019/11/05 20:19:18 maxv Exp $ */
/*-
* Copyright (c) 1992, 1993
@@ -37,6 +37,7 @@
#ifdef _KERNEL_OPT
#include "opt_diagnostic.h"
#include "opt_kasan.h"
+#include "opt_kcsan.h"
#endif
#include <sys/types.h>
@@ -376,11 +377,18 @@ void *kasan_memset(void *, int, size_t);
#define memcpy(d, s, l) kasan_memcpy(d, s, l)
#define memcmp(a, b, l) kasan_memcmp(a, b, l)
#define memset(d, v, l) kasan_memset(d, v, l)
+#elif defined(_KERNEL) && defined(KCSAN)
+void *kcsan_memcpy(void *, const void *, size_t);
+int kcsan_memcmp(const void *, const void *, size_t);
+void *kcsan_memset(void *, int, size_t);
+#define memcpy(d, s, l) kcsan_memcpy(d, s, l)
+#define memcmp(a, b, l) kcsan_memcmp(a, b, l)
+#define memset(d, v, l) kcsan_memset(d, v, l)
#else
#define memcpy(d, s, l) __builtin_memcpy(d, s, l)
#define memcmp(a, b, l) __builtin_memcmp(a, b, l)
#define memset(d, v, l) __builtin_memset(d, v, l)
-#endif /* _KERNEL && KASAN */
+#endif
#endif
char *strcpy(char *, const char *);
@@ -396,11 +404,18 @@ size_t kasan_strlen(const char *);
#define strcpy(d, s) kasan_strcpy(d, s)
#define strcmp(a, b) kasan_strcmp(a, b)
#define strlen(a) kasan_strlen(a)
+#elif defined(_KERNEL) && defined(KCSAN)
+char *kcsan_strcpy(char *, const char *);
+int kcsan_strcmp(const char *, const char *);
+size_t kcsan_strlen(const char *);
+#define strcpy(d, s) kcsan_strcpy(d, s)
+#define strcmp(a, b) kcsan_strcmp(a, b)
+#define strlen(a) kcsan_strlen(a)
#else
#define strcpy(d, s) __builtin_strcpy(d, s)
#define strcmp(a, b) __builtin_strcmp(a, b)
#define strlen(a) __builtin_strlen(a)
-#endif /* _KERNEL && KASAN */
+#endif
#endif
/* Functions for which we always use built-ins. */
@@ -442,6 +457,9 @@ void *memmove(void *, const void *, size
#if defined(_KERNEL) && defined(KASAN)
void *kasan_memmove(void *, const void *, size_t);
#define memmove(d, s, l) kasan_memmove(d, s, l)
+#elif defined(_KERNEL) && defined(KCSAN)
+void *kcsan_memmove(void *, const void *, size_t);
+#define memmove(d, s, l) kcsan_memmove(d, s, l)
#endif
int pmatch(const char *, const char *, const char **);
Index: src/sys/sys/atomic.h
diff -u src/sys/sys/atomic.h:1.15 src/sys/sys/atomic.h:1.16
--- src/sys/sys/atomic.h:1.15 Wed Sep 11 14:56:25 2019
+++ src/sys/sys/atomic.h Tue Nov 5 20:19:18 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.h,v 1.15 2019/09/11 14:56:25 christos Exp $ */
+/* $NetBSD: atomic.h,v 1.16 2019/11/05 20:19:18 maxv Exp $ */
/*-
* Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
@@ -39,6 +39,7 @@
#if defined(_KERNEL) && defined(_KERNEL_OPT)
#include "opt_kasan.h"
+#include "opt_kcsan.h"
#endif
#if defined(KASAN)
@@ -62,6 +63,27 @@
#define ATOMIC_PROTO_INC(name, tret, targ1) \
void kasan_atomic_inc_##name(volatile targ1 *); \
tret kasan_atomic_inc_##name##_nv(volatile targ1 *)
+#elif defined(KCSAN)
+#define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
+ void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
+ tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2)
+#define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
+ void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
+ tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2)
+#define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
+ void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
+ tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2)
+#define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
+ tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
+ tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
+#define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
+ tret kcsan_atomic_swap_##name(volatile targ1 *, targ2)
+#define ATOMIC_PROTO_DEC(name, tret, targ1) \
+ void kcsan_atomic_dec_##name(volatile targ1 *); \
+ tret kcsan_atomic_dec_##name##_nv(volatile targ1 *)
+#define ATOMIC_PROTO_INC(name, tret, targ1) \
+ void kcsan_atomic_inc_##name(volatile targ1 *); \
+ tret kcsan_atomic_inc_##name##_nv(volatile targ1 *)
#else
#define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
void atomic_add_##name(volatile targ1 *, targ2); \
@@ -213,6 +235,68 @@ __END_DECLS
#define atomic_inc_ulong_nv kasan_atomic_inc_ulong_nv
#define atomic_inc_ptr_nv kasan_atomic_inc_ptr_nv
#define atomic_inc_64_nv kasan_atomic_inc_64_nv
+#elif defined(KCSAN)
+#define atomic_add_32 kcsan_atomic_add_32
+#define atomic_add_int kcsan_atomic_add_int
+#define atomic_add_long kcsan_atomic_add_long
+#define atomic_add_ptr kcsan_atomic_add_ptr
+#define atomic_add_64 kcsan_atomic_add_64
+#define atomic_add_32_nv kcsan_atomic_add_32_nv
+#define atomic_add_int_nv kcsan_atomic_add_int_nv
+#define atomic_add_long_nv kcsan_atomic_add_long_nv
+#define atomic_add_ptr_nv kcsan_atomic_add_ptr_nv
+#define atomic_add_64_nv kcsan_atomic_add_64_nv
+#define atomic_and_32 kcsan_atomic_and_32
+#define atomic_and_uint kcsan_atomic_and_uint
+#define atomic_and_ulong kcsan_atomic_and_ulong
+#define atomic_and_64 kcsan_atomic_and_64
+#define atomic_and_32_nv kcsan_atomic_and_32_nv
+#define atomic_and_uint_nv kcsan_atomic_and_uint_nv
+#define atomic_and_ulong_nv kcsan_atomic_and_ulong_nv
+#define atomic_and_64_nv kcsan_atomic_and_64_nv
+#define atomic_or_32 kcsan_atomic_or_32
+#define atomic_or_uint kcsan_atomic_or_uint
+#define atomic_or_ulong kcsan_atomic_or_ulong
+#define atomic_or_64 kcsan_atomic_or_64
+#define atomic_or_32_nv kcsan_atomic_or_32_nv
+#define atomic_or_uint_nv kcsan_atomic_or_uint_nv
+#define atomic_or_ulong_nv kcsan_atomic_or_ulong_nv
+#define atomic_or_64_nv kcsan_atomic_or_64_nv
+#define atomic_cas_32 kcsan_atomic_cas_32
+#define atomic_cas_uint kcsan_atomic_cas_uint
+#define atomic_cas_ulong kcsan_atomic_cas_ulong
+#define atomic_cas_ptr kcsan_atomic_cas_ptr
+#define atomic_cas_64 kcsan_atomic_cas_64
+#define atomic_cas_32_ni kcsan_atomic_cas_32_ni
+#define atomic_cas_uint_ni kcsan_atomic_cas_uint_ni
+#define atomic_cas_ulong_ni kcsan_atomic_cas_ulong_ni
+#define atomic_cas_ptr_ni kcsan_atomic_cas_ptr_ni
+#define atomic_cas_64_ni kcsan_atomic_cas_64_ni
+#define atomic_swap_32 kcsan_atomic_swap_32
+#define atomic_swap_uint kcsan_atomic_swap_uint
+#define atomic_swap_ulong kcsan_atomic_swap_ulong
+#define atomic_swap_ptr kcsan_atomic_swap_ptr
+#define atomic_swap_64 kcsan_atomic_swap_64
+#define atomic_dec_32 kcsan_atomic_dec_32
+#define atomic_dec_uint kcsan_atomic_dec_uint
+#define atomic_dec_ulong kcsan_atomic_dec_ulong
+#define atomic_dec_ptr kcsan_atomic_dec_ptr
+#define atomic_dec_64 kcsan_atomic_dec_64
+#define atomic_dec_32_nv kcsan_atomic_dec_32_nv
+#define atomic_dec_uint_nv kcsan_atomic_dec_uint_nv
+#define atomic_dec_ulong_nv kcsan_atomic_dec_ulong_nv
+#define atomic_dec_ptr_nv kcsan_atomic_dec_ptr_nv
+#define atomic_dec_64_nv kcsan_atomic_dec_64_nv
+#define atomic_inc_32 kcsan_atomic_inc_32
+#define atomic_inc_uint kcsan_atomic_inc_uint
+#define atomic_inc_ulong kcsan_atomic_inc_ulong
+#define atomic_inc_ptr kcsan_atomic_inc_ptr
+#define atomic_inc_64 kcsan_atomic_inc_64
+#define atomic_inc_32_nv kcsan_atomic_inc_32_nv
+#define atomic_inc_uint_nv kcsan_atomic_inc_uint_nv
+#define atomic_inc_ulong_nv kcsan_atomic_inc_ulong_nv
+#define atomic_inc_ptr_nv kcsan_atomic_inc_ptr_nv
+#define atomic_inc_64_nv kcsan_atomic_inc_64_nv
#endif
#endif /* ! _SYS_ATOMIC_H_ */
Index: src/sys/sys/bus_proto.h
diff -u src/sys/sys/bus_proto.h:1.9 src/sys/sys/bus_proto.h:1.10
--- src/sys/sys/bus_proto.h:1.9 Sun Sep 22 10:35:12 2019
+++ src/sys/sys/bus_proto.h Tue Nov 5 20:19:18 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: bus_proto.h,v 1.9 2019/09/22 10:35:12 maxv Exp $ */
+/* $NetBSD: bus_proto.h,v 1.10 2019/11/05 20:19:18 maxv Exp $ */
/*-
* Copyright (c) 1996, 1997, 1998, 2001, 2007 The NetBSD Foundation, Inc.
@@ -66,6 +66,7 @@
#ifdef _KERNEL_OPT
#include "opt_kasan.h"
+#include "opt_kcsan.h"
#endif
/*
@@ -158,6 +159,32 @@ void kasan_bus_space_read_region_stream_
#define bus_space_read_region_stream_2 kasan_bus_space_read_region_stream_2
#define bus_space_read_region_stream_4 kasan_bus_space_read_region_stream_4
#define bus_space_read_region_stream_8 kasan_bus_space_read_region_stream_8
+#elif defined(KCSAN)
+#define BUS_SPACE_READ_MEM_PROTOS(bytes, bits) \
+void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, uint##bits##_t *, bus_size_t); \
+void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, uint##bits##_t *, bus_size_t); \
+void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);
+#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1
+#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2
+#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4
+#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8
+#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1
+#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2
+#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4
+#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8
+#define bus_space_read_region_1 kcsan_bus_space_read_region_1
+#define bus_space_read_region_2 kcsan_bus_space_read_region_2
+#define bus_space_read_region_4 kcsan_bus_space_read_region_4
+#define bus_space_read_region_8 kcsan_bus_space_read_region_8
+#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1
+#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2
+#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4
+#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8
#else
#define BUS_SPACE_READ_MEM_PROTOS(bytes, bits) \
void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
@@ -221,6 +248,32 @@ void kasan_bus_space_write_region_stream
#define bus_space_write_region_stream_2 kasan_bus_space_write_region_stream_2
#define bus_space_write_region_stream_4 kasan_bus_space_write_region_stream_4
#define bus_space_write_region_stream_8 kasan_bus_space_write_region_stream_8
+#elif defined(KCSAN)
+#define BUS_SPACE_WRITE_MEM_PROTOS(bytes, bits) \
+void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, const uint##bits##_t *, bus_size_t); \
+void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t); \
+void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, const uint##bits##_t *, bus_size_t); \
+void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);
+#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1
+#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2
+#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4
+#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8
+#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1
+#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2
+#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4
+#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8
+#define bus_space_write_region_1 kcsan_bus_space_write_region_1
+#define bus_space_write_region_2 kcsan_bus_space_write_region_2
+#define bus_space_write_region_4 kcsan_bus_space_write_region_4
+#define bus_space_write_region_8 kcsan_bus_space_write_region_8
+#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1
+#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2
+#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4
+#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8
#else
#define BUS_SPACE_WRITE_MEM_PROTOS(bytes, bits) \
void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
Index: src/sys/sys/cdefs.h
diff -u src/sys/sys/cdefs.h:1.147 src/sys/sys/cdefs.h:1.148
--- src/sys/sys/cdefs.h:1.147 Wed Oct 16 18:29:49 2019
+++ src/sys/sys/cdefs.h Tue Nov 5 20:19:18 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cdefs.h,v 1.147 2019/10/16 18:29:49 christos Exp $ */
+/* $NetBSD: cdefs.h,v 1.148 2019/11/05 20:19:18 maxv Exp $ */
/* * Copyright (c) 1991, 1993
* The Regents of the University of California. All rights reserved.
@@ -39,6 +39,7 @@
#ifdef _KERNEL_OPT
#include "opt_diagnostic.h"
#include "opt_kasan.h"
+#include "opt_kcsan.h"
#endif
/*
@@ -341,6 +342,12 @@
#define __noasan /* nothing */
#endif
+#if __GNUC_PREREQ__(4, 9) && defined(KCSAN)
+#define __nocsan __attribute__((no_sanitize_thread))
+#else
+#define __nocsan /* nothing */
+#endif
+
#if defined(__clang__)
#define __noubsan __attribute__((no_sanitize("undefined")))
#elif __GNUC_PREREQ__(4, 9)
Index: src/sys/sys/systm.h
diff -u src/sys/sys/systm.h:1.287 src/sys/sys/systm.h:1.288
--- src/sys/sys/systm.h:1.287 Thu Oct 10 13:45:14 2019
+++ src/sys/sys/systm.h Tue Nov 5 20:19:18 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: systm.h,v 1.287 2019/10/10 13:45:14 maxv Exp $ */
+/* $NetBSD: systm.h,v 1.288 2019/11/05 20:19:18 maxv Exp $ */
/*-
* Copyright (c) 1982, 1988, 1991, 1993
@@ -44,6 +44,7 @@
#include "opt_multiprocessor.h"
#include "opt_gprof.h"
#include "opt_kasan.h"
+#include "opt_kcsan.h"
#include "opt_kleak.h"
#include "opt_wsdisplay_compat.h"
#endif
@@ -267,6 +268,9 @@ void tablefull(const char *, const char
#if defined(_KERNEL) && defined(KASAN)
int kasan_kcopy(const void *, void *, size_t);
#define kcopy kasan_kcopy
+#elif defined(_KERNEL) && defined(KCSAN)
+int kcsan_kcopy(const void *, void *, size_t);
+#define kcopy kcsan_kcopy
#else
int kcopy(const void *, void *, size_t);
#endif
@@ -286,6 +290,15 @@ int kasan_copyin(const void *, void *, s
#define copyinstr kasan_copyinstr
#define copyoutstr kasan_copyoutstr
#define copyin kasan_copyin
+#elif defined(_KERNEL) && defined(KCSAN)
+int kcsan_copystr(const void *, void *, size_t, size_t *);
+int kcsan_copyinstr(const void *, void *, size_t, size_t *);
+int kcsan_copyoutstr(const void *, void *, size_t, size_t *);
+int kcsan_copyin(const void *, void *, size_t);
+#define copystr kcsan_copystr
+#define copyinstr kcsan_copyinstr
+#define copyoutstr kcsan_copyoutstr
+#define copyin kcsan_copyin
#else
int copystr(const void *, void *, size_t, size_t *);
int copyinstr(const void *, void *, size_t, size_t *);
Added files:
Index: src/sys/arch/amd64/include/csan.h
diff -u /dev/null src/sys/arch/amd64/include/csan.h:1.1
--- /dev/null Tue Nov 5 20:19:18 2019
+++ src/sys/arch/amd64/include/csan.h Tue Nov 5 20:19:17 2019
@@ -0,0 +1,119 @@
+/* $NetBSD: csan.h,v 1.1 2019/11/05 20:19:17 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/ksyms.h>
+#include <x86/cpufunc.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return (x86_read_psl() & PSL_I) == 0;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+ *state = x86_read_psl();
+ x86_disable_intr();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+ x86_write_psl(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static inline bool
+__md_unwind_end(const char *name)
+{
+ if (!strcmp(name, "syscall") ||
+ !strcmp(name, "alltraps") ||
+ !strcmp(name, "handle_syscall") ||
+ !strncmp(name, "Xtrap", 5) ||
+ !strncmp(name, "Xintr", 5) ||
+ !strncmp(name, "Xhandle", 7) ||
+ !strncmp(name, "Xresume", 7) ||
+ !strncmp(name, "Xstray", 6) ||
+ !strncmp(name, "Xhold", 5) ||
+ !strncmp(name, "Xrecurse", 8) ||
+ !strcmp(name, "Xdoreti") ||
+ !strncmp(name, "Xsoft", 5)) {
+ return true;
+ }
+
+ return false;
+}
+
+static void
+kcsan_md_unwind(void)
+{
+ uint64_t *rbp, rip;
+ const char *mod;
+ const char *sym;
+ size_t nsym;
+ int error;
+
+ rbp = (uint64_t *)__builtin_frame_address(0);
+ nsym = 0;
+
+ while (1) {
+ /* 8(%rbp) contains the saved %rip. */
+ rip = *(rbp + 1);
+
+ if (rip < KERNBASE) {
+ break;
+ }
+ error = ksyms_getname(&mod, &sym, (vaddr_t)rip, KSYMS_PROC);
+ if (error) {
+ break;
+ }
+ printf("#%zu %p in %s <%s>\n", nsym, (void *)rip, sym, mod);
+ if (__md_unwind_end(sym)) {
+ break;
+ }
+
+ rbp = (uint64_t *)*(rbp);
+ if (rbp == 0) {
+ break;
+ }
+ nsym++;
+
+ if (nsym >= 15) {
+ break;
+ }
+ }
+}
Index: src/sys/kern/subr_csan.c
diff -u /dev/null src/sys/kern/subr_csan.c:1.1
--- /dev/null Tue Nov 5 20:19:18 2019
+++ src/sys/kern/subr_csan.c Tue Nov 5 20:19:17 2019
@@ -0,0 +1,744 @@
+/* $NetBSD: subr_csan.c,v 1.1 2019/11/05 20:19:17 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.1 2019/11/05 20:19:17 maxv Exp $");
+
+#include "opt_kcsan.h"
+
+#include <sys/param.h>
+#include <sys/device.h>
+#include <sys/kernel.h>
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/csan.h>
+#include <sys/cpu.h>
+
+#ifdef KCSAN_PANIC
+#define REPORT panic
+#else
+#define REPORT printf
+#endif
+
+typedef struct {
+ uintptr_t addr;
+ uint32_t size;
+ bool write:1;
+ bool atomic:1;
+ uintptr_t pc;
+} csan_cell_t;
+
+typedef struct {
+ bool inited;
+ uint32_t cnt;
+ csan_cell_t cell;
+} csan_cpu_t;
+
+static csan_cpu_t kcsan_cpus[MAXCPUS];
+static bool kcsan_enabled __read_mostly;
+
+#define __RET_ADDR (uintptr_t)__builtin_return_address(0)
+
+#define KCSAN_NACCESSES 2000
+#define KCSAN_DELAY 10000 /* 10 milliseconds */
+
+/* -------------------------------------------------------------------------- */
+
+/* The MD code. */
+#include <machine/csan.h>
+
+/* -------------------------------------------------------------------------- */
+
+void
+kcsan_init(void)
+{
+ kcsan_enabled = true;
+}
+
+void
+kcsan_cpu_init(struct cpu_info *ci)
+{
+ kcsan_cpus[cpu_index(ci)].inited = true;
+}
+
+/* -------------------------------------------------------------------------- */
+
+static inline void
+kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu)
+{
+ const char *newsym, *oldsym;
+
+ if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) {
+ newsym = "Unknown";
+ }
+ if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) {
+ oldsym = "Unknown";
+ }
+ REPORT("CSan: Racy Access "
+ "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] "
+ "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n",
+ newcpu,
+ (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
+ (void *)new->addr, new->size, (void *)new->pc, newsym,
+ oldcpu,
+ (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
+ (void *)old->addr, old->size, (void *)old->pc, oldsym);
+ kcsan_md_unwind();
+}
+
+static inline bool
+kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
+{
+ if (new->write && !new->atomic)
+ return false;
+ if (old->write && !old->atomic)
+ return false;
+ return true;
+}
+
+static inline void
+kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
+{
+ csan_cell_t old, new;
+ csan_cpu_t *cpu;
+ uint64_t intr;
+ size_t i;
+
+ if (__predict_false(!kcsan_enabled))
+ return;
+
+ new.addr = addr;
+ new.size = size;
+ new.write = write;
+ new.atomic = atomic;
+ new.pc = pc;
+
+ for (i = 0; i < ncpu; i++) {
+ __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
+
+ if (old.addr + old.size <= new.addr)
+ continue;
+ if (new.addr + new.size <= old.addr)
+ continue;
+ if (__predict_true(!old.write && !new.write))
+ continue;
+ if (__predict_true(kcsan_access_is_atomic(&new, &old)))
+ continue;
+
+ kcsan_report(&new, cpu_number(), &old, i);
+ break;
+ }
+
+ if (__predict_false(!kcsan_md_is_avail()))
+ return;
+
+ kcsan_md_disable_intrs(&intr);
+
+ cpu = &kcsan_cpus[cpu_number()];
+ if (__predict_false(!cpu->inited))
+ goto out;
+ cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
+ if (__predict_true(cpu->cnt != 0))
+ goto out;
+
+ __builtin_memcpy(&cpu->cell, &new, sizeof(new));
+ kcsan_md_delay(KCSAN_DELAY);
+ __builtin_memset(&cpu->cell, 0, sizeof(new));
+
+out:
+ kcsan_md_enable_intrs(&intr);
+}
+
+#define CSAN_READ(size) \
+ void __tsan_read##size(uintptr_t); \
+ void __tsan_read##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, false, false, __RET_ADDR); \
+ }
+
+CSAN_READ(1)
+CSAN_READ(2)
+CSAN_READ(4)
+CSAN_READ(8)
+CSAN_READ(16)
+
+#define CSAN_WRITE(size) \
+ void __tsan_write##size(uintptr_t); \
+ void __tsan_write##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, true, false, __RET_ADDR); \
+ }
+
+CSAN_WRITE(1)
+CSAN_WRITE(2)
+CSAN_WRITE(4)
+CSAN_WRITE(8)
+CSAN_WRITE(16)
+
+void __tsan_read_range(uintptr_t, size_t);
+void __tsan_write_range(uintptr_t, size_t);
+
+void
+__tsan_read_range(uintptr_t addr, size_t size)
+{
+ kcsan_access(addr, size, false, false, __RET_ADDR);
+}
+
+void
+__tsan_write_range(uintptr_t addr, size_t size)
+{
+ kcsan_access(addr, size, true, false, __RET_ADDR);
+}
+
+void __tsan_init(void);
+void __tsan_func_entry(void *);
+void __tsan_func_exit(void);
+
+void
+__tsan_init(void)
+{
+}
+
+void
+__tsan_func_entry(void *call_pc)
+{
+}
+
+void
+__tsan_func_exit(void)
+{
+}
+
+/* -------------------------------------------------------------------------- */
+
+void *
+kcsan_memcpy(void *dst, const void *src, size_t len)
+{
+ kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
+ return __builtin_memcpy(dst, src, len);
+}
+
+int
+kcsan_memcmp(const void *b1, const void *b2, size_t len)
+{
+ kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
+ return __builtin_memcmp(b1, b2, len);
+}
+
+void *
+kcsan_memset(void *b, int c, size_t len)
+{
+ kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
+ return __builtin_memset(b, c, len);
+}
+
+void *
+kcsan_memmove(void *dst, const void *src, size_t len)
+{
+ kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
+ return __builtin_memmove(dst, src, len);
+}
+
+char *
+kcsan_strcpy(char *dst, const char *src)
+{
+ char *save = dst;
+
+ while (1) {
+ kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
+ *dst = *src;
+ if (*src == '\0')
+ break;
+ src++, dst++;
+ }
+
+ return save;
+}
+
+int
+kcsan_strcmp(const char *s1, const char *s2)
+{
+ while (1) {
+ kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
+ if (*s1 != *s2)
+ break;
+ if (*s1 == '\0')
+ return 0;
+ s1++, s2++;
+ }
+
+ return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
+}
+
+size_t
+kcsan_strlen(const char *str)
+{
+ const char *s;
+
+ s = str;
+ while (1) {
+ kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
+ if (*s == '\0')
+ break;
+ s++;
+ }
+
+ return (s - str);
+}
+
+#undef kcopy
+#undef copystr
+#undef copyinstr
+#undef copyoutstr
+#undef copyin
+
+int kcsan_kcopy(const void *, void *, size_t);
+int kcsan_copystr(const void *, void *, size_t, size_t *);
+int kcsan_copyinstr(const void *, void *, size_t, size_t *);
+int kcsan_copyoutstr(const void *, void *, size_t, size_t *);
+int kcsan_copyin(const void *, void *, size_t);
+int kcopy(const void *, void *, size_t);
+int copystr(const void *, void *, size_t, size_t *);
+int copyinstr(const void *, void *, size_t, size_t *);
+int copyoutstr(const void *, void *, size_t, size_t *);
+int copyin(const void *, void *, size_t);
+
+int
+kcsan_kcopy(const void *src, void *dst, size_t len)
+{
+ kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
+ return kcopy(src, dst, len);
+}
+
+int
+kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
+{
+ kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
+ return copystr(kfaddr, kdaddr, len, done);
+}
+
+int
+kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
+{
+ kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
+ return copyin(uaddr, kaddr, len);
+}
+
+int
+kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
+{
+ kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
+ return copyinstr(uaddr, kaddr, len, done);
+}
+
+int
+kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
+{
+ kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
+ return copyoutstr(kaddr, uaddr, len, done);
+}
+
+/* -------------------------------------------------------------------------- */
+
+#undef atomic_add_32
+#undef atomic_add_int
+#undef atomic_add_long
+#undef atomic_add_ptr
+#undef atomic_add_64
+#undef atomic_add_32_nv
+#undef atomic_add_int_nv
+#undef atomic_add_long_nv
+#undef atomic_add_ptr_nv
+#undef atomic_add_64_nv
+#undef atomic_and_32
+#undef atomic_and_uint
+#undef atomic_and_ulong
+#undef atomic_and_64
+#undef atomic_and_32_nv
+#undef atomic_and_uint_nv
+#undef atomic_and_ulong_nv
+#undef atomic_and_64_nv
+#undef atomic_or_32
+#undef atomic_or_uint
+#undef atomic_or_ulong
+#undef atomic_or_64
+#undef atomic_or_32_nv
+#undef atomic_or_uint_nv
+#undef atomic_or_ulong_nv
+#undef atomic_or_64_nv
+#undef atomic_cas_32
+#undef atomic_cas_uint
+#undef atomic_cas_ulong
+#undef atomic_cas_ptr
+#undef atomic_cas_64
+#undef atomic_cas_32_ni
+#undef atomic_cas_uint_ni
+#undef atomic_cas_ulong_ni
+#undef atomic_cas_ptr_ni
+#undef atomic_cas_64_ni
+#undef atomic_swap_32
+#undef atomic_swap_uint
+#undef atomic_swap_ulong
+#undef atomic_swap_ptr
+#undef atomic_swap_64
+#undef atomic_dec_32
+#undef atomic_dec_uint
+#undef atomic_dec_ulong
+#undef atomic_dec_ptr
+#undef atomic_dec_64
+#undef atomic_dec_32_nv
+#undef atomic_dec_uint_nv
+#undef atomic_dec_ulong_nv
+#undef atomic_dec_ptr_nv
+#undef atomic_dec_64_nv
+#undef atomic_inc_32
+#undef atomic_inc_uint
+#undef atomic_inc_ulong
+#undef atomic_inc_ptr
+#undef atomic_inc_64
+#undef atomic_inc_32_nv
+#undef atomic_inc_uint_nv
+#undef atomic_inc_ulong_nv
+#undef atomic_inc_ptr_nv
+#undef atomic_inc_64_nv
+
+#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
+ void atomic_add_##name(volatile targ1 *, targ2); \
+ void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
+ void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ atomic_add_##name(ptr, val); \
+ } \
+ tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
+ tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
+ tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_add_##name##_nv(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
+ void atomic_and_##name(volatile targ1 *, targ2); \
+ void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
+ void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ atomic_and_##name(ptr, val); \
+ } \
+ tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
+ tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
+ tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_and_##name##_nv(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
+ void atomic_or_##name(volatile targ1 *, targ2); \
+ void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
+ void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ atomic_or_##name(ptr, val); \
+ } \
+ tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
+ tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
+ tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_or_##name##_nv(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
+ tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
+ tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
+ tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_cas_##name(ptr, exp, new); \
+ } \
+ tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
+ tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
+ tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_cas_##name##_ni(ptr, exp, new); \
+ }
+
+#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
+ tret atomic_swap_##name(volatile targ1 *, targ2); \
+ tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \
+ tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_swap_##name(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
+ void atomic_dec_##name(volatile targ1 *); \
+ void kcsan_atomic_dec_##name(volatile targ1 *); \
+ void kcsan_atomic_dec_##name(volatile targ1 *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ atomic_dec_##name(ptr); \
+ } \
+ tret atomic_dec_##name##_nv(volatile targ1 *); \
+ tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \
+ tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_dec_##name##_nv(ptr); \
+ }
+
+#define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
+ void atomic_inc_##name(volatile targ1 *); \
+ void kcsan_atomic_inc_##name(volatile targ1 *); \
+ void kcsan_atomic_inc_##name(volatile targ1 *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ atomic_inc_##name(ptr); \
+ } \
+ tret atomic_inc_##name##_nv(volatile targ1 *); \
+ tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \
+ tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
+ __RET_ADDR); \
+ return atomic_inc_##name##_nv(ptr); \
+ }
+
+CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
+CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
+CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
+CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
+CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
+
+CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
+CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
+CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
+CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
+
+CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
+CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
+CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
+CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
+
+CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
+CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
+CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
+CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
+CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
+
+CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
+CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
+CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
+CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
+CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
+
+CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
+CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
+CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
+CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
+CSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
+
+CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
+CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
+CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
+CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
+CSAN_ATOMIC_FUNC_INC(ptr, void *, void);
+
+/* -------------------------------------------------------------------------- */
+
+#include <sys/bus.h>
+
+#undef bus_space_read_multi_1
+#undef bus_space_read_multi_2
+#undef bus_space_read_multi_4
+#undef bus_space_read_multi_8
+#undef bus_space_read_multi_stream_1
+#undef bus_space_read_multi_stream_2
+#undef bus_space_read_multi_stream_4
+#undef bus_space_read_multi_stream_8
+#undef bus_space_read_region_1
+#undef bus_space_read_region_2
+#undef bus_space_read_region_4
+#undef bus_space_read_region_8
+#undef bus_space_read_region_stream_1
+#undef bus_space_read_region_stream_2
+#undef bus_space_read_region_stream_4
+#undef bus_space_read_region_stream_8
+#undef bus_space_write_multi_1
+#undef bus_space_write_multi_2
+#undef bus_space_write_multi_4
+#undef bus_space_write_multi_8
+#undef bus_space_write_multi_stream_1
+#undef bus_space_write_multi_stream_2
+#undef bus_space_write_multi_stream_4
+#undef bus_space_write_multi_stream_8
+#undef bus_space_write_region_1
+#undef bus_space_write_region_2
+#undef bus_space_write_region_4
+#undef bus_space_write_region_8
+#undef bus_space_write_region_stream_1
+#undef bus_space_write_region_stream_2
+#undef bus_space_write_region_stream_4
+#undef bus_space_write_region_stream_8
+
+#define CSAN_BUS_READ_FUNC(bytes, bits) \
+ void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ false, false, __RET_ADDR); \
+ bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
+ } \
+ void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ false, false, __RET_ADDR); \
+ bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
+ } \
+ void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ false, false, __RET_ADDR); \
+ bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
+ } \
+ void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ false, false, __RET_ADDR); \
+ bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
+ }
+
+#define CSAN_BUS_WRITE_FUNC(bytes, bits) \
+ void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
+ bus_size_t, const uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
+ void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ true, false, __RET_ADDR); \
+ bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
+ } \
+ void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
+ void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
+ void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ true, false, __RET_ADDR); \
+ bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
+ } \
+ void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
+ bus_size_t, const uint##bits##_t *, bus_size_t); \
+ void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
+ void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ true, false, __RET_ADDR); \
+ bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
+ } \
+ void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
+ void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
+ void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
+ true, false, __RET_ADDR); \
+ bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
+ }
+
+CSAN_BUS_READ_FUNC(1, 8)
+CSAN_BUS_READ_FUNC(2, 16)
+CSAN_BUS_READ_FUNC(4, 32)
+CSAN_BUS_READ_FUNC(8, 64)
+
+CSAN_BUS_WRITE_FUNC(1, 8)
+CSAN_BUS_WRITE_FUNC(2, 16)
+CSAN_BUS_WRITE_FUNC(4, 32)
+CSAN_BUS_WRITE_FUNC(8, 64)
Index: src/sys/sys/csan.h
diff -u /dev/null src/sys/sys/csan.h:1.1
--- /dev/null Tue Nov 5 20:19:18 2019
+++ src/sys/sys/csan.h Tue Nov 5 20:19:18 2019
@@ -0,0 +1,49 @@
+/* $NetBSD: csan.h,v 1.1 2019/11/05 20:19:18 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SYS_CSAN_H_
+#define _SYS_CSAN_H_
+
+#ifdef _KERNEL_OPT
+#include "opt_kcsan.h"
+#endif
+
+#include <sys/types.h>
+
+#ifdef KCSAN
+void kcsan_init(void);
+void kcsan_cpu_init(struct cpu_info *);
+#else
+#define kcsan_init() __nothing
+#define kcsan_cpu_init(ci) __nothing
+#endif
+
+#endif /* !_SYS_CSAN_H_ */
