Author: andrew Date: Thu Nov 21 11:22:08 2019 New Revision: 354942 URL: https://svnweb.freebsd.org/changeset/base/354942
Log: Port the NetBSD KCSAN runtime to FreeBSD. Update the NetBSD Kernel Concurrency Sanitizer (KCSAN) runtime to work in the FreeBSD kernel. It is a useful tool for finding data races between threads executing on different CPUs. This can be enabled by enabling KCSAN in the kernel config, or by using the GENERIC-KCSAN amd64 kernel. It works on amd64 and arm64, however the later needs a compiler change to allow -fsanitize=thread that KCSAN uses. Sponsored by: DARPA, AFRL Differential Revision: https://reviews.freebsd.org/D22315 Added: head/sys/amd64/conf/GENERIC-KCSAN (contents, props changed) head/sys/amd64/include/csan.h (contents, props changed) head/sys/arm64/include/csan.h (contents, props changed) head/sys/sys/_cscan_atomic.h (contents, props changed) head/sys/sys/_cscan_bus.h (contents, props changed) Modified: head/sys/amd64/amd64/copyout.c head/sys/amd64/amd64/machdep.c head/sys/amd64/conf/GENERIC head/sys/amd64/include/atomic.h head/sys/arm64/arm64/bus_machdep.c head/sys/arm64/arm64/copystr.c head/sys/arm64/arm64/machdep.c head/sys/arm64/arm64/mp_machdep.c head/sys/arm64/conf/GENERIC head/sys/arm64/include/atomic.h head/sys/arm64/include/bus.h head/sys/conf/files head/sys/conf/files.arm64 head/sys/conf/kern.post.mk head/sys/conf/kern.pre.mk head/sys/conf/options head/sys/kern/subr_csan.c (contents, props changed) head/sys/kern/vfs_aio.c head/sys/libkern/strcmp.c head/sys/libkern/strcpy.c head/sys/libkern/strlen.c head/sys/modules/Makefile head/sys/sys/csan.h (contents, props changed) head/sys/sys/libkern.h head/sys/sys/systm.h head/sys/x86/include/bus.h head/sys/x86/x86/bus_machdep.c head/sys/x86/x86/mp_x86.c Modified: head/sys/amd64/amd64/copyout.c ============================================================================== --- head/sys/amd64/amd64/copyout.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/amd64/amd64/copyout.c Thu Nov 21 11:22:08 2019 (r354942) @@ -146,6 +146,10 @@ DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_l casueword_smap : casueword_nosmap); } +#undef copyinstr +#undef copyin +#undef copyout + int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); int copyinstr_smap(const void *udaddr, void *kaddr, size_t len, Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/amd64/amd64/machdep.c Thu Nov 21 11:22:08 2019 (r354942) @@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$"); #include <sys/callout.h> #include <sys/cons.h> #include <sys/cpu.h> +#include <sys/csan.h> #include <sys/efi.h> #include <sys/eventhandler.h> #include <sys/exec.h> @@ -1899,6 +1900,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) cpu_probe_amdc1e(); + kcsan_cpu_init(0); + #ifdef FDT x86_init_fdt(); #endif @@ -2722,6 +2725,40 @@ outb_(u_short port, u_char data) void *memset_std(void *buf, int c, size_t len); void *memset_erms(void *buf, int c, size_t len); +void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, + size_t len); + +#ifdef KCSAN +/* + * These fail to build as ifuncs when used with KCSAN. + */ +void * +memset(void *buf, int c, size_t len) +{ + + return memset_std(buf, c, len); +} + +void * +memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len) +{ + + return memmove_std(dst, src, len); +} + +void * +memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len) +{ + + return memcpy_std(dst, src, len); +} +#else DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) { @@ -2729,10 +2766,6 @@ DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) memset_erms : memset_std); } -void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, - size_t len); -void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, - size_t len); DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, size_t)) { @@ -2741,16 +2774,13 @@ DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, cons memmove_erms : memmove_std); } -void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, - size_t len); -void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, - size_t len); DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ? memcpy_erms : memcpy_std); } +#endif void pagezero_std(void *addr); void pagezero_erms(void *addr); Modified: head/sys/amd64/conf/GENERIC ============================================================================== --- head/sys/amd64/conf/GENERIC Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/amd64/conf/GENERIC Thu Nov 21 11:22:08 2019 (r354942) @@ -106,6 +106,7 @@ options VERBOSE_SYSINIT=0 # Support debug.verbose_sys #options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer +#options KCSAN # Kernel Concurrency Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Added: head/sys/amd64/conf/GENERIC-KCSAN ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/amd64/conf/GENERIC-KCSAN Thu Nov 21 11:22:08 2019 (r354942) @@ -0,0 +1,33 @@ +# +# GENERIC-KCSAN -- Kernel Concurrency Sanitizer kernel configuration file +# for FreeBSD/amd64 +# +# This configuration file removes several debugging options, including +# WITNESS and INVARIANTS checking, which are known to have significant +# performance impact on running systems. When benchmarking new features +# this kernel should be used instead of the standard GENERIC. +# This kernel configuration should never appear outside of the HEAD +# of the FreeBSD tree. +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# https://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +include GENERIC + +ident GENERIC-KCSAN + +options KCSAN Modified: head/sys/amd64/include/atomic.h ============================================================================== --- head/sys/amd64/include/atomic.h Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/amd64/include/atomic.h Thu Nov 21 11:22:08 2019 (r354942) @@ -57,6 +57,20 @@ #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") +#ifdef _KERNEL +/* + * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). + * + * The open-coded number is used instead of the symbolic expression to + * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. + * An assertion in amd64/vm_machdep.c ensures that the value is correct. + */ +#define OFFSETOF_MONITORBUF 0x100 +#endif + +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include <sys/_cscan_atomic.h> +#else #include <sys/atomic_common.h> /* @@ -345,15 +359,6 @@ atomic_testandclear_long(volatile u_long *p, u_int v) #if defined(_KERNEL) -/* - * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). - * - * The open-coded number is used instead of the symbolic expression to - * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. - * An assertion in amd64/vm_machdep.c ensures that the value is correct. - */ -#define OFFSETOF_MONITORBUF 0x100 - #if defined(SMP) || defined(KLD_MODULE) static __inline void __storeload_barrier(void) @@ -678,5 +683,7 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ + +#endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* !_MACHINE_ATOMIC_H_ */ Added: head/sys/amd64/include/csan.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/amd64/include/csan.h Thu Nov 21 11:22:08 2019 (r354942) @@ -0,0 +1,67 @@ +/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include <machine/cpufunc.h> +#include <machine/stack.h> +#include <machine/vmparam.h> + +static inline bool +kcsan_md_is_avail(void) +{ + return true; +} + +static inline void +kcsan_md_disable_intrs(uint64_t *state) +{ + + *state = intr_disable(); +} + +static inline void +kcsan_md_enable_intrs(uint64_t *state) +{ + + intr_restore(*state); +} + +static inline void +kcsan_md_delay(uint64_t us) +{ + DELAY(us); +} + +static void +kcsan_md_unwind(void) +{ +} Modified: head/sys/arm64/arm64/bus_machdep.c ============================================================================== --- head/sys/arm64/arm64/bus_machdep.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/arm64/bus_machdep.c Thu Nov 21 11:22:08 2019 (r354942) @@ -25,6 +25,8 @@ * */ +#define KCSAN_RUNTIME + #include "opt_platform.h" #include <sys/param.h> Modified: head/sys/arm64/arm64/copystr.c ============================================================================== --- head/sys/arm64/arm64/copystr.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/arm64/copystr.c Thu Nov 21 11:22:08 2019 (r354942) @@ -32,7 +32,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> int -copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len, +(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len, size_t * __restrict lencopied) { const char *src; Modified: head/sys/arm64/arm64/machdep.c ============================================================================== --- head/sys/arm64/arm64/machdep.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/arm64/machdep.c Thu Nov 21 11:22:08 2019 (r354942) @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/cons.h> #include <sys/cpu.h> +#include <sys/csan.h> #include <sys/devmap.h> #include <sys/efi.h> #include <sys/exec.h> @@ -1208,6 +1209,8 @@ initarm(struct arm64_bootparams *abp) dbg_init(); kdb_init(); pan_enable(); + + kcsan_cpu_init(0); env = kern_getenv("kernelname"); if (env != NULL) Modified: head/sys/arm64/arm64/mp_machdep.c ============================================================================== --- head/sys/arm64/arm64/mp_machdep.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/arm64/mp_machdep.c Thu Nov 21 11:22:08 2019 (r354942) @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/bus.h> #include <sys/cpu.h> +#include <sys/csan.h> #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/malloc.h> @@ -252,6 +253,8 @@ init_secondary(uint64_t cpu) } mtx_unlock_spin(&ap_boot_mtx); + + kcsan_cpu_init(cpu); /* Enter the scheduler */ sched_throw(NULL); Modified: head/sys/arm64/conf/GENERIC ============================================================================== --- head/sys/arm64/conf/GENERIC Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/conf/GENERIC Thu Nov 21 11:22:08 2019 (r354942) @@ -100,6 +100,7 @@ options VERBOSE_SYSINIT=0 # Support debug.verbose_sys #options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer +#options KCSAN # Kernel Concurrency Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Modified: head/sys/arm64/include/atomic.h ============================================================================== --- head/sys/arm64/include/atomic.h Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/include/atomic.h Thu Nov 21 11:22:08 2019 (r354942) @@ -29,8 +29,6 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ -#include <sys/atomic_common.h> - #define isb() __asm __volatile("isb" : : : "memory") /* @@ -55,6 +53,12 @@ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include <sys/_cscan_atomic.h> +#else + +#include <sys/atomic_common.h> + #define ATOMIC_OP(op, asm_op, bar, a, l) \ static __inline void \ atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \ @@ -600,6 +604,8 @@ atomic_thread_fence_seq_cst(void) dmb(sy); } + +#endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* _MACHINE_ATOMIC_H_ */ Modified: head/sys/arm64/include/bus.h ============================================================================== --- head/sys/arm64/include/bus.h Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/arm64/include/bus.h Thu Nov 21 11:22:08 2019 (r354942) @@ -89,6 +89,9 @@ #define BUS_SPACE_BARRIER_READ 0x01 #define BUS_SPACE_BARRIER_WRITE 0x02 +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include <sys/_cscan_bus.h> +#else struct bus_space { /* cookie */ @@ -463,6 +466,8 @@ struct bus_space { __bs_copy(4, t, h1, o1, h2, o2, c) #define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ __bs_copy(8, t, h1, o1, h2, o2, c) + +#endif #include <machine/bus_dma.h> Added: head/sys/arm64/include/csan.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/arm64/include/csan.h Thu Nov 21 11:22:08 2019 (r354942) @@ -0,0 +1,104 @@ +/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include <machine/cpufunc.h> +#include <machine/stack.h> +#include <machine/vmparam.h> + +static inline bool +kcsan_md_is_avail(void) +{ + return true; +} + +static inline void +kcsan_md_disable_intrs(uint64_t *state) +{ + + *state = intr_disable(); +} + +static inline void +kcsan_md_enable_intrs(uint64_t *state) +{ + + intr_restore(*state); +} + +static inline void +kcsan_md_delay(uint64_t us) +{ + DELAY(us); +} + +static void +kcsan_md_unwind(void) +{ +#ifdef DDB + c_db_sym_t sym; + db_expr_t offset; + const char *symname; +#endif + struct unwind_state frame; + uint64_t sp; + int nsym; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + frame.sp = sp; + frame.fp = (uint64_t)__builtin_frame_address(0); + frame.pc = (uint64_t)kcsan_md_unwind; + nsym = 0; + + while (1) { + unwind_frame(&frame); + if (!INKERNEL((vm_offset_t)frame.fp) || + !INKERNEL((vm_offset_t)frame.pc)) + break; + +#ifdef DDB + sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC, + &offset); + db_symbol_values(sym, &symname, NULL); + printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc, + symname, offset); +#else + printf("#%d %p\n", nsym, (void *)frame.pc); +#endif + nsym++; + + if (nsym >= 15) { + break; + } + } +} Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/conf/files Thu Nov 21 11:22:08 2019 (r354942) @@ -3805,6 +3805,8 @@ kern/subr_compressor.c standard \ kern/subr_coverage.c optional coverage \ compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_counter.c standard +kern/subr_csan.c optional kcsan \ + compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_early.c standard Modified: head/sys/conf/files.arm64 ============================================================================== --- head/sys/conf/files.arm64 Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/conf/files.arm64 Thu Nov 21 11:22:08 2019 (r354942) @@ -286,8 +286,10 @@ kern/pic_if.m optional intrng kern/subr_devmap.c standard kern/subr_intr.c optional intrng libkern/bcmp.c standard -libkern/memcmp.c standard -libkern/memset.c standard +libkern/memcmp.c standard \ + compile-with "${NORMAL_C:N-fsanitize*}" +libkern/memset.c standard \ + compile-with "${NORMAL_C:N-fsanitize*}" libkern/arm64/crc32c_armv8.S standard cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" Modified: head/sys/conf/kern.post.mk ============================================================================== --- head/sys/conf/kern.post.mk Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/conf/kern.post.mk Thu Nov 21 11:22:08 2019 (r354942) @@ -38,6 +38,10 @@ MKMODULESENV+= WITH_CTF="${WITH_CTF}" MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}" .endif +.if defined(KCSAN_ENABLED) +MKMODULESENV+= KCSAN_ENABLED="yes" +.endif + .if defined(SAN_CFLAGS) MKMODULESENV+= SAN_CFLAGS="${SAN_CFLAGS}" .endif Modified: head/sys/conf/kern.pre.mk ============================================================================== --- head/sys/conf/kern.pre.mk Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/conf/kern.pre.mk Thu Nov 21 11:22:08 2019 (r354942) @@ -117,6 +117,11 @@ PROF= -pg .endif DEFINED_PROF= ${PROF} +KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo +.if !empty(KCSAN_ENABLED) +SAN_CFLAGS+= -fsanitize=thread +.endif + KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo .if !empty(KUBSAN_ENABLED) SAN_CFLAGS+= -fsanitize=undefined Modified: head/sys/conf/options ============================================================================== --- head/sys/conf/options Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/conf/options Thu Nov 21 11:22:08 2019 (r354942) @@ -230,6 +230,7 @@ ZSTDIO opt_zstdio.h # Sanitizers COVERAGE opt_global.h KCOV +KCSAN opt_global.h KUBSAN opt_global.h # POSIX kernel options Modified: head/sys/kern/subr_csan.c ============================================================================== --- head/sys/kern/subr_csan.c Thu Nov 21 08:20:05 2019 (r354941) +++ head/sys/kern/subr_csan.c Thu Nov 21 11:22:08 2019 (r354942) @@ -3,6 +3,7 @@ /* * Copyright (c) 2019 The NetBSD Foundation, Inc. * All rights reserved. + * Copyright (c) 2019 Andrew Turner * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. @@ -29,19 +30,26 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#define KCSAN_RUNTIME + +#include "opt_ddb.h" + #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $"); +__FBSDID("$FreeBSD$"); #include <sys/param.h> -#include <sys/device.h> #include <sys/kernel.h> -#include <sys/param.h> +#include <sys/bus.h> #include <sys/conf.h> -#include <sys/systm.h> -#include <sys/types.h> -#include <sys/csan.h> #include <sys/cpu.h> +#include <sys/csan.h> +#include <sys/proc.h> +#include <sys/smp.h> +#include <sys/systm.h> +#include <ddb/ddb.h> +#include <ddb/db_sym.h> + #ifdef KCSAN_PANIC #define REPORT panic #else @@ -62,7 +70,7 @@ typedef struct { csan_cell_t cell; } csan_cpu_t; -static csan_cpu_t kcsan_cpus[MAXCPUS]; +static csan_cpu_t kcsan_cpus[MAXCPU]; static bool kcsan_enabled __read_mostly; #define __RET_ADDR (uintptr_t)__builtin_return_address(0) @@ -77,34 +85,43 @@ static bool kcsan_enabled __read_mostly; /* -------------------------------------------------------------------------- */ -void -kcsan_init(void) +static void +kcsan_enable(void *dummy __unused) { + + printf("Enabling KCSCAN, expect reduced performance.\n"); kcsan_enabled = true; } +SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); void -kcsan_cpu_init(struct cpu_info *ci) +kcsan_cpu_init(u_int cpu) { - kcsan_cpus[cpu_index(ci)].inited = true; + kcsan_cpus[cpu].inited = true; } /* -------------------------------------------------------------------------- */ static inline void -kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) +kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) { const char *newsym, *oldsym; +#ifdef DDB + c_db_sym_t sym; + db_expr_t offset; - if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { - newsym = "Unknown"; - } - if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { - oldsym = "Unknown"; - } + sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); + db_symbol_values(sym, &newsym, NULL); + + sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); + db_symbol_values(sym, &oldsym, NULL); +#else + newsym = ""; + oldsym = ""; +#endif REPORT("CSan: Racy Access " - "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " - "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", + "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " + "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", newcpu, (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), (void *)new->addr, new->size, (void *)new->pc, newsym, @@ -134,8 +151,6 @@ kcsan_access(uintptr_t addr, size_t size, bool write, if (__predict_false(!kcsan_enabled)) return; - if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) - return; new.addr = addr; new.size = size; @@ -143,7 +158,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, new.atomic = atomic; new.pc = pc; - for (i = 0; i < ncpu; i++) { + CPU_FOREACH(i) { __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); if (old.addr + old.size <= new.addr) @@ -155,7 +170,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, if (__predict_true(kcsan_access_is_atomic(&new, &old))) continue; - kcsan_report(&new, cpu_number(), &old, i); + kcsan_report(&new, PCPU_GET(cpuid), &old, i); break; } @@ -164,7 +179,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, kcsan_md_disable_intrs(&intr); - cpu = &kcsan_cpus[cpu_number()]; + cpu = &kcsan_cpus[PCPU_GET(cpuid)]; if (__predict_false(!cpu->inited)) goto out; cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; @@ -184,6 +199,11 @@ out: void __tsan_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ + } \ + void __tsan_unaligned_read##size(uintptr_t); \ + void __tsan_unaligned_read##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, false, false, __RET_ADDR); \ } CSAN_READ(1) @@ -197,6 +217,11 @@ CSAN_READ(16) void __tsan_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ + } \ + void __tsan_unaligned_write##size(uintptr_t); \ + void __tsan_unaligned_write##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, true, false, __RET_ADDR); \ } CSAN_WRITE(1) @@ -321,35 +346,14 @@ kcsan_strlen(const char *str) return (s - str); } -#undef kcopy #undef copystr -#undef copyinstr -#undef copyoutstr #undef copyin +#undef copyin_nofault +#undef copyinstr #undef copyout +#undef copyout_nofault -int kcsan_kcopy(const void *, void *, size_t); -int kcsan_copystr(const void *, void *, size_t, size_t *); -int kcsan_copyinstr(const void *, void *, size_t, size_t *); -int kcsan_copyoutstr(const void *, void *, size_t, size_t *); -int kcsan_copyin(const void *, void *, size_t); -int kcsan_copyout(const void *, void *, size_t); -int kcopy(const void *, void *, size_t); -int copystr(const void *, void *, size_t, size_t *); -int copyinstr(const void *, void *, size_t, size_t *); -int copyoutstr(const void *, void *, size_t, size_t *); -int copyin(const void *, void *, size_t); -int copyout(const void *, void *, size_t); - int -kcsan_kcopy(const void *src, void *dst, size_t len) -{ - kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); - kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); - return kcopy(src, dst, len); -} - -int kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) { kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR); @@ -364,13 +368,6 @@ kcsan_copyin(const void *uaddr, void *kaddr, size_t le } int -kcsan_copyout(const void *kaddr, void *uaddr, size_t len) -{ - kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); - return copyout(kaddr, uaddr, len); -} - -int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); @@ -378,377 +375,477 @@ kcsan_copyinstr(const void *uaddr, void *kaddr, size_t } int -kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) +kcsan_copyout(const void *kaddr, void *uaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); - return copyoutstr(kaddr, uaddr, len, done); + return copyout(kaddr, uaddr, len); } /* -------------------------------------------------------------------------- */ -#undef atomic_add_32 -#undef atomic_add_int -#undef atomic_add_long -#undef atomic_add_ptr -#undef atomic_add_64 -#undef atomic_add_32_nv -#undef atomic_add_int_nv -#undef atomic_add_long_nv -#undef atomic_add_ptr_nv -#undef atomic_add_64_nv -#undef atomic_and_32 -#undef atomic_and_uint -#undef atomic_and_ulong -#undef atomic_and_64 -#undef atomic_and_32_nv -#undef atomic_and_uint_nv -#undef atomic_and_ulong_nv -#undef atomic_and_64_nv -#undef atomic_or_32 -#undef atomic_or_uint -#undef atomic_or_ulong -#undef atomic_or_64 -#undef atomic_or_32_nv -#undef atomic_or_uint_nv -#undef atomic_or_ulong_nv -#undef atomic_or_64_nv -#undef atomic_cas_32 -#undef atomic_cas_uint -#undef atomic_cas_ulong -#undef atomic_cas_ptr -#undef atomic_cas_64 -#undef atomic_cas_32_ni -#undef atomic_cas_uint_ni -#undef atomic_cas_ulong_ni -#undef atomic_cas_ptr_ni -#undef atomic_cas_64_ni -#undef atomic_swap_32 -#undef atomic_swap_uint -#undef atomic_swap_ulong -#undef atomic_swap_ptr -#undef atomic_swap_64 -#undef atomic_dec_32 -#undef atomic_dec_uint -#undef atomic_dec_ulong -#undef atomic_dec_ptr -#undef atomic_dec_64 -#undef atomic_dec_32_nv -#undef atomic_dec_uint_nv -#undef atomic_dec_ulong_nv -#undef atomic_dec_ptr_nv -#undef atomic_dec_64_nv -#undef atomic_inc_32 -#undef atomic_inc_uint -#undef atomic_inc_ulong -#undef atomic_inc_ptr -#undef atomic_inc_64 -#undef atomic_inc_32_nv -#undef atomic_inc_uint_nv -#undef atomic_inc_ulong_nv -#undef atomic_inc_ptr_nv -#undef atomic_inc_64_nv +#include <machine/atomic.h> +#include <sys/_cscan_atomic.h> -#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ - void atomic_add_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_add_##name(ptr, val); \ - } \ - tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_add_##name##_nv(ptr, val); \ +#define _CSAN_ATOMIC_FUNC_ADD(name, type) \ + void kcsan_atomic_add_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_add_##name(ptr, val); \ } -#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ - void atomic_and_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_and_##name(ptr, val); \ - } \ - tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_and_##name##_nv(ptr, val); \ +#define CSAN_ATOMIC_FUNC_ADD(name, type) \ + _CSAN_ATOMIC_FUNC_ADD(name, type) \ + _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_clear_##name(ptr, val); \ } -#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ - void atomic_or_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_or_##name(ptr, val); \ - } \ - tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_or_##name##_nv(ptr, val); \ +#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ + type val2) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_cmpset_##name(ptr, val1, val2)); \ } -#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ - tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_cas_##name(ptr, exp, new); \ - } \ - tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_cas_##name##_ni(ptr, exp, new); \ +#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ + type val2) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_fcmpset_##name(ptr, val1, val2)); \ } -#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ - tret atomic_swap_##name(volatile targ1 *, targ2); \ - tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ - tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_swap_##name(ptr, val); \ +#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) + +#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ + type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_fetchadd_##name(ptr, val)); \ } -#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ - void atomic_dec_##name(volatile targ1 *); \ - void kcsan_atomic_dec_##name(volatile targ1 *); \ - void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
