Author: andrew Date: Tue Jan 29 11:04:17 2019 New Revision: 343550 URL: https://svnweb.freebsd.org/changeset/base/343550
Log: Extract the coverage sanitizer KPI to a new file. This will allow multiple consumers of the coverage data to be compiled into the kernel together. The only requirement is only one can be registered at a given point in time, however it is expected they will only register when the coverage data is needed. A new kernel conflig option COVERAGE is added. This will allow kcov to become a module that can be loaded as needed, or compiled into the kernel. While here clean up the #include style a little. Reviewed by: kib Sponsored by: DARPA, AFRL Differential Revision: https://reviews.freebsd.org/D18955 Added: head/sys/kern/subr_coverage.c - copied, changed from r343547, head/sys/kern/kern_kcov.c head/sys/sys/coverage.h - copied, changed from r343547, head/sys/sys/kcov.h Modified: head/sys/amd64/conf/GENERIC head/sys/amd64/conf/GENERIC-NODEBUG head/sys/arm64/conf/GENERIC head/sys/arm64/conf/GENERIC-NODEBUG head/sys/conf/files head/sys/conf/kern.pre.mk head/sys/conf/options head/sys/kern/kern_kcov.c head/sys/sys/kcov.h Modified: head/sys/amd64/conf/GENERIC ============================================================================== --- head/sys/amd64/conf/GENERIC Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/amd64/conf/GENERIC Tue Jan 29 11:04:17 2019 (r343550) @@ -100,9 +100,12 @@ options WITNESS # Enable checks to detect deadlocks options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default + +# Kernel Sanitizers +#options COVERAGE # Generic kernel coverage. Used by KCOV +#options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer -#options KCOV # Kernel Coverage Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Modified: head/sys/amd64/conf/GENERIC-NODEBUG ============================================================================== --- head/sys/amd64/conf/GENERIC-NODEBUG Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/amd64/conf/GENERIC-NODEBUG Tue Jan 29 11:04:17 2019 (r343550) @@ -37,4 +37,5 @@ nooptions WITNESS_SKIPSPIN nooptions BUF_TRACKING nooptions DEADLKRES nooptions FULL_BUF_TRACKING - +nooptions COVERAGE +nooptions KCOV Modified: head/sys/arm64/conf/GENERIC ============================================================================== --- head/sys/arm64/conf/GENERIC Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/arm64/conf/GENERIC Tue Jan 29 11:04:17 2019 (r343550) @@ -92,9 +92,12 @@ options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) options ALT_BREAK_TO_DEBUGGER # Enter debugger on keyboard escape sequence options USB_DEBUG # enable debug msgs options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default + +# Kernel Sanitizers +#options COVERAGE # Generic kernel coverage. Used by KCOV +#options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer -#options KCOV # Kernel Coverage Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Modified: head/sys/arm64/conf/GENERIC-NODEBUG ============================================================================== --- head/sys/arm64/conf/GENERIC-NODEBUG Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/arm64/conf/GENERIC-NODEBUG Tue Jan 29 11:04:17 2019 (r343550) @@ -36,3 +36,5 @@ nooptions WITNESS nooptions WITNESS_SKIPSPIN nooptions DEADLKRES nooptions USB_DEBUG +nooptions COVERAGE +nooptions KCOV Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/conf/files Tue Jan 29 11:04:17 2019 (r343550) @@ -3883,6 +3883,8 @@ kern/subr_capability.c standard kern/subr_clock.c standard kern/subr_compressor.c standard \ compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd" +kern/subr_coverage.c optional coverage \ + compile-with "${NORMAL_C} -fno-sanitize=all" kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard Modified: head/sys/conf/kern.pre.mk ============================================================================== --- head/sys/conf/kern.pre.mk Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/conf/kern.pre.mk Tue Jan 29 11:04:17 2019 (r343550) @@ -118,8 +118,8 @@ KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; ec SAN_CFLAGS+= -fsanitize=undefined .endif -KCOV_ENABLED!= grep KCOV opt_kcov.h || true ; echo -.if !empty(KCOV_ENABLED) +COVERAGE_ENABLED!= grep COVERAGE opt_global.h || true ; echo +.if !empty(COVERAGE_ENABLED) SAN_CFLAGS+= -fsanitize-coverage=trace-pc,trace-cmp .endif Modified: head/sys/conf/options ============================================================================== --- head/sys/conf/options Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/conf/options Tue Jan 29 11:04:17 2019 (r343550) @@ -57,7 +57,6 @@ DDB_CTF opt_ddb.h DDB_NUMSYM opt_ddb.h FULL_BUF_TRACKING opt_global.h GDB -KCOV opt_kcov.h KDB opt_global.h KDB_TRACE opt_kdb.h KDB_UNATTENDED opt_kdb.h @@ -234,6 +233,8 @@ VERBOSE_SYSINIT ZSTDIO opt_zstdio.h # Sanitizers +COVERAGE opt_global.h +KCOV KUBSAN opt_global.h # POSIX kernel options Modified: head/sys/kern/kern_kcov.c ============================================================================== --- head/sys/kern/kern_kcov.c Tue Jan 29 10:28:50 2019 (r343549) +++ head/sys/kern/kern_kcov.c Tue Jan 29 11:04:17 2019 (r343550) @@ -39,29 +39,26 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> +#include <sys/systm.h> #include <sys/conf.h> -#include <sys/file.h> #include <sys/kcov.h> #include <sys/kernel.h> +#include <sys/limits.h> #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mman.h> #include <sys/mutex.h> #include <sys/proc.h> #include <sys/rwlock.h> -#include <sys/stat.h> #include <sys/sysctl.h> -#include <sys/systm.h> -#include <sys/types.h> #include <vm/vm.h> +#include <vm/pmap.h> #include <vm/vm_extern.h> #include <vm/vm_object.h> #include <vm/vm_page.h> #include <vm/vm_pager.h> -#include <vm/pmap.h> - MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); #define KCOV_ELEMENT_SIZE sizeof(uint64_t) @@ -138,17 +135,6 @@ static d_close_t kcov_close; static d_mmap_single_t kcov_mmap_single; static d_ioctl_t kcov_ioctl; -void __sanitizer_cov_trace_pc(void); -void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); -void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); -void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t); -void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t); -void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t); -void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t); -void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t); -void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); -void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); - static int kcov_alloc(struct kcov_info *info, size_t entries); static void kcov_init(const void *unused); @@ -169,6 +155,7 @@ SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG "Maximum number of entries in the kcov buffer"); static struct mtx kcov_lock; +static int active_count; static struct kcov_info * get_kinfo(struct thread *td) @@ -197,25 +184,13 @@ get_kinfo(struct thread *td) return (info); } -/* - * Main entry point. A call to this function will be inserted - * at every edge, and if coverage is enabled for the thread - * this function will add the PC to the buffer. - */ -void -__sanitizer_cov_trace_pc(void) +static void +trace_pc(uintptr_t ret) { struct thread *td; struct kcov_info *info; uint64_t *buf, index; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return; - td = curthread; info = get_kinfo(td); if (info == NULL) @@ -237,7 +212,7 @@ __sanitizer_cov_trace_pc(void) if (index + 2 > info->entries) return; - buf[index + 1] = (uint64_t)__builtin_return_address(0); + buf[index + 1] = ret; buf[0] = index + 1; } @@ -248,13 +223,6 @@ trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, struct kcov_info *info; uint64_t *buf, index; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return (false); - td = curthread; info = get_kinfo(td); if (info == NULL) @@ -287,109 +255,7 @@ trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, return (true); } -void -__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - /* - * val is the switch operand - * cases[0] is the number of case constants - * cases[1] is the size of val in bits - * cases[2..n] are the case constants - */ -void -__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) -{ - uint64_t i, count, ret, type; - - count = cases[0]; - ret = (uint64_t)__builtin_return_address(0); - - switch (cases[1]) { - case 8: - type = KCOV_CMP_SIZE(0); - break; - case 16: - type = KCOV_CMP_SIZE(1); - break; - case 32: - type = KCOV_CMP_SIZE(2); - break; - case 64: - type = KCOV_CMP_SIZE(3); - break; - default: - return; - } - - val |= KCOV_CMP_CONST; - - for (i = 0; i < count; i++) - if (!trace_cmp(type, val, cases[i + 2], ret)) - return; -} - -/* * The fd is being closed, cleanup everything we can. */ static void @@ -456,6 +322,7 @@ kcov_close(struct cdev *dev, int fflag, int devtype, s struct kcov_info *info; int error; + if ((error = devfs_get_cdevpriv((void **)&info)) != 0) return (error); @@ -571,6 +438,16 @@ kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, error = EINVAL; break; } + + /* Lets hope nobody opens this 2 billion times */ + KASSERT(active_count < INT_MAX, + ("%s: Open too many times", __func__)); + active_count++; + if (active_count == 1) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } + KASSERT(info->thread == NULL, ("Enabling kcov when already enabled")); info->thread = td; @@ -589,6 +466,13 @@ kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, error = EINVAL; break; } + KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); + active_count--; + if (active_count == 0) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } + td->td_kcov_info = NULL; atomic_store_int(&info->state, KCOV_STATE_READY); /* @@ -618,6 +502,12 @@ kcov_thread_dtor(void *arg __unused, struct thread *td return; mtx_lock_spin(&kcov_lock); + KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); + active_count--; + if (active_count == 0) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } td->td_kcov_info = NULL; if (info->state != KCOV_STATE_DYING) { /* @@ -673,4 +563,4 @@ kcov_init(const void *unused) EVENTHANDLER_PRI_ANY); } -SYSINIT(kcovdev, SI_SUB_DEVFS, SI_ORDER_ANY, kcov_init, NULL); +SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL); Copied and modified: head/sys/kern/subr_coverage.c (from r343547, head/sys/kern/kern_kcov.c) ============================================================================== --- head/sys/kern/kern_kcov.c Tue Jan 29 09:22:20 2019 (r343547, copy source) +++ head/sys/kern/subr_coverage.c Tue Jan 29 11:04:17 2019 (r343550) @@ -39,105 +39,10 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> -#include <sys/conf.h> -#include <sys/file.h> -#include <sys/kcov.h> -#include <sys/kernel.h> -#include <sys/lock.h> -#include <sys/malloc.h> -#include <sys/mman.h> -#include <sys/mutex.h> -#include <sys/proc.h> -#include <sys/rwlock.h> -#include <sys/stat.h> -#include <sys/sysctl.h> -#include <sys/systm.h> -#include <sys/types.h> +#include <sys/coverage.h> -#include <vm/vm.h> -#include <vm/vm_extern.h> -#include <vm/vm_object.h> -#include <vm/vm_page.h> -#include <vm/vm_pager.h> +#include <machine/atomic.h> -#include <vm/pmap.h> - -MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); - -#define KCOV_ELEMENT_SIZE sizeof(uint64_t) - -/* - * To know what the code can safely perform at any point in time we use a - * state machine. In the normal case the state transitions are: - * - * OPEN -> READY -> RUNNING -> DYING - * | | ^ | ^ ^ - * | | +--------+ | | - * | +-------------------+ | - * +-----------------------------+ - * - * The states are: - * OPEN: The kcov fd has been opened, but no buffer is available to store - * coverage data. - * READY: The buffer to store coverage data has been allocated. Userspace - * can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When - * this has been set the buffer can be written to by the kernel, - * and mmaped by userspace. - * RUNNING: The coverage probes are able to store coverage data in the buffer. - * This is entered with ioctl(fd, KIOENABLE, mode);. The READY state - * can be exited by ioctl(fd, KIODISABLE); or exiting the thread to - * return to the READY state to allow tracing to be reused, or by - * closing the kcov fd to enter the DYING state. - * DYING: The fd has been closed. All states can enter into this state when - * userspace closes the kcov fd. - * - * We need to be careful when moving into and out of the RUNNING state. As - * an interrupt may happen while this is happening the ordering of memory - * operations is important so struct kcov_info is valid for the tracing - * functions. - * - * When moving into the RUNNING state prior stores to struct kcov_info need - * to be observed before the state is set. This allows for interrupts that - * may call into one of the coverage functions to fire at any point while - * being enabled and see a consistent struct kcov_info. - * - * When moving out of the RUNNING state any later stores to struct kcov_info - * need to be observed after the state is set. As with entering this is to - * present a consistent struct kcov_info to interrupts. - */ -typedef enum { - KCOV_STATE_INVALID, - KCOV_STATE_OPEN, /* The device is open, but with no buffer */ - KCOV_STATE_READY, /* The buffer has been allocated */ - KCOV_STATE_RUNNING, /* Recording trace data */ - KCOV_STATE_DYING, /* The fd was closed */ -} kcov_state_t; - -/* - * (l) Set while holding the kcov_lock mutex and not in the RUNNING state. - * (o) Only set once while in the OPEN state. Cleaned up while in the DYING - * state, and with no thread associated with the struct kcov_info. - * (s) Set atomically to enter or exit the RUNNING state, non-atomically - * otherwise. See above for a description of the other constraints while - * moving into or out of the RUNNING state. - */ -struct kcov_info { - struct thread *thread; /* (l) */ - vm_object_t bufobj; /* (o) */ - vm_offset_t kvaddr; /* (o) */ - size_t entries; /* (o) */ - size_t bufsize; /* (o) */ - kcov_state_t state; /* (s) */ - int mode; /* (l) */ - bool mmap; -}; - -/* Prototypes */ -static d_open_t kcov_open; -static d_close_t kcov_close; -static d_mmap_single_t kcov_mmap_single; -static d_ioctl_t kcov_ioctl; - void __sanitizer_cov_trace_pc(void); void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); @@ -149,52 +54,35 @@ void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32 void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); -static int kcov_alloc(struct kcov_info *info, size_t entries); -static void kcov_init(const void *unused); +static cov_trace_pc_t cov_trace_pc; +static cov_trace_cmp_t cov_trace_cmp; -static struct cdevsw kcov_cdevsw = { - .d_version = D_VERSION, - .d_open = kcov_open, - .d_close = kcov_close, - .d_mmap_single = kcov_mmap_single, - .d_ioctl = kcov_ioctl, - .d_name = "kcov", -}; +void +cov_register_pc(cov_trace_pc_t trace_pc) +{ -SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW, 0, "Kernel coverage"); + atomic_store_ptr(&cov_trace_pc, trace_pc); +} -static u_int kcov_max_entries = KCOV_MAXENTRIES; -SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW, - &kcov_max_entries, 0, - "Maximum number of entries in the kcov buffer"); +void +cov_unregister_pc(void) +{ -static struct mtx kcov_lock; + atomic_store_ptr(&cov_trace_pc, NULL); +} -static struct kcov_info * -get_kinfo(struct thread *td) +void +cov_register_cmp(cov_trace_cmp_t trace_cmp) { - struct kcov_info *info; - /* We might have a NULL thread when releasing the secondary CPUs */ - if (td == NULL) - return (NULL); + atomic_store_ptr(&cov_trace_cmp, trace_cmp); +} - /* - * We are in an interrupt, stop tracing as it is not explicitly - * part of a syscall. - */ - if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL) - return (NULL); +void +cov_unregister_cmp(void) +{ - /* - * If info is NULL or the state is not running we are not tracing. - */ - info = td->td_kcov_info; - if (info == NULL || - atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING) - return (NULL); - - return (info); + atomic_store_ptr(&cov_trace_cmp, NULL); } /* @@ -205,150 +93,104 @@ get_kinfo(struct thread *td) void __sanitizer_cov_trace_pc(void) { - struct thread *td; - struct kcov_info *info; - uint64_t *buf, index; + cov_trace_pc_t trace_pc; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return; - - td = curthread; - info = get_kinfo(td); - if (info == NULL) - return; - - /* - * Check we are in the PC-trace mode. - */ - if (info->mode != KCOV_MODE_TRACE_PC) - return; - - KASSERT(info->kvaddr != 0, - ("__sanitizer_cov_trace_pc: NULL buf while running")); - - buf = (uint64_t *)info->kvaddr; - - /* The first entry of the buffer holds the index */ - index = buf[0]; - if (index + 2 > info->entries) - return; - - buf[index + 1] = (uint64_t)__builtin_return_address(0); - buf[0] = index + 1; + trace_pc = (cov_trace_pc_t)atomic_load_ptr(&cov_trace_pc); + if (trace_pc != NULL) + trace_pc((uint64_t)__builtin_return_address(0)); } -static bool -trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret) -{ - struct thread *td; - struct kcov_info *info; - uint64_t *buf, index; - - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return (false); - - td = curthread; - info = get_kinfo(td); - if (info == NULL) - return (false); - - /* - * Check we are in the comparison-trace mode. - */ - if (info->mode != KCOV_MODE_TRACE_CMP) - return (false); - - KASSERT(info->kvaddr != 0, - ("__sanitizer_cov_trace_pc: NULL buf while running")); - - buf = (uint64_t *)info->kvaddr; - - /* The first entry of the buffer holds the index */ - index = buf[0]; - - /* Check we have space to store all elements */ - if (index * 4 + 4 + 1 > info->entries) - return (false); - - buf[index * 4 + 1] = type; - buf[index * 4 + 2] = arg1; - buf[index * 4 + 3] = arg2; - buf[index * 4 + 4] = ret; - buf[0] = index + 1; - - return (true); -} - +/* + * Comparison entry points. When the kernel performs a comparison + * operation the compiler inserts a call to one of the following + * functions to record the operation. + */ void __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(0), arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(1), arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(2), arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(3), arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(0) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(1) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(2) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) { + cov_trace_cmp_t trace_cmp; - trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(3) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); } /* @@ -361,316 +203,35 @@ void __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) { uint64_t i, count, ret, type; + cov_trace_cmp_t trace_cmp; + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp == NULL) + return; + count = cases[0]; ret = (uint64_t)__builtin_return_address(0); switch (cases[1]) { case 8: - type = KCOV_CMP_SIZE(0); + type = COV_CMP_SIZE(0); break; case 16: - type = KCOV_CMP_SIZE(1); + type = COV_CMP_SIZE(1); break; case 32: - type = KCOV_CMP_SIZE(2); + type = COV_CMP_SIZE(2); break; case 64: - type = KCOV_CMP_SIZE(3); + type = COV_CMP_SIZE(3); break; default: return; } - val |= KCOV_CMP_CONST; + val |= COV_CMP_CONST; for (i = 0; i < count; i++) if (!trace_cmp(type, val, cases[i + 2], ret)) return; } - -/* - * The fd is being closed, cleanup everything we can. - */ -static void -kcov_mmap_cleanup(void *arg) -{ - struct kcov_info *info = arg; - struct thread *thread; - - mtx_lock_spin(&kcov_lock); - /* - * Move to KCOV_STATE_DYING to stop adding new entries. - * - * If the thread is running we need to wait until thread exit to - * clean up as it may currently be adding a new entry. If this is - * the case being in KCOV_STATE_DYING will signal that the buffer - * needs to be cleaned up. - */ - atomic_store_int(&info->state, KCOV_STATE_DYING); - atomic_thread_fence_seq_cst(); - thread = info->thread; - mtx_unlock_spin(&kcov_lock); - - if (thread != NULL) - return; - - /* - * We can safely clean up the info struct as it is in the - * KCOV_STATE_DYING state with no thread associated. - * - * The KCOV_STATE_DYING stops new threads from using it. - * The lack of a thread means nothing is currently using the buffers. - */ - - if (info->kvaddr != 0) { - pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE); - kva_free(info->kvaddr, info->bufsize); - } - if (info->bufobj != NULL && !info->mmap) - vm_object_deallocate(info->bufobj); - free(info, M_KCOV_INFO); -} - -static int -kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td) -{ - struct kcov_info *info; - int error; - - info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK); - info->state = KCOV_STATE_OPEN; - info->thread = NULL; - info->mode = -1; - info->mmap = false; - - if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0) - kcov_mmap_cleanup(info); - - return (error); -} - -static int -kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td) -{ - struct kcov_info *info; - int error; - - if ((error = devfs_get_cdevpriv((void **)&info)) != 0) - return (error); - - KASSERT(info != NULL, ("kcov_close with no kcov_info structure")); - - /* Trying to close, but haven't disabled */ - if (info->state == KCOV_STATE_RUNNING) - return (EBUSY); - - return (0); -} - -static int -kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size, - struct vm_object **object, int nprot) -{ - struct kcov_info *info; - int error; - - if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) != - (PROT_READ | PROT_WRITE)) - return (EINVAL); - - if ((error = devfs_get_cdevpriv((void **)&info)) != 0) - return (error); - - if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries || - info->mmap != false) - return (EINVAL); - - info->mmap = true; - *offset = 0; - *object = info->bufobj; - return (0); -} - -static int -kcov_alloc(struct kcov_info *info, size_t entries) -{ - size_t n, pages; - vm_page_t *m; - - KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer")); - KASSERT(info->state == KCOV_STATE_OPEN, - ("kcov_alloc: Not in open state (%x)", info->state)); - - if (entries < 2 || entries > kcov_max_entries) - return (EINVAL); - - /* Align to page size so mmap can't access other kernel memory */ - info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE); - pages = info->bufsize / PAGE_SIZE; - - if ((info->kvaddr = kva_alloc(info->bufsize)) == 0) - return (ENOMEM); - - info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize, - PROT_READ | PROT_WRITE, 0, curthread->td_ucred); - - m = malloc(sizeof(*m) * pages, M_TEMP, M_WAITOK); - VM_OBJECT_WLOCK(info->bufobj); - for (n = 0; n < pages; n++) { - m[n] = vm_page_grab(info->bufobj, n, - VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_WIRED); - m[n]->valid = VM_PAGE_BITS_ALL; - } - VM_OBJECT_WUNLOCK(info->bufobj); - pmap_qenter(info->kvaddr, m, pages); - free(m, M_TEMP); - - info->entries = entries; - - return (0); -} - -static int -kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused, - struct thread *td) -{ - struct kcov_info *info; - int mode, error; - - if ((error = devfs_get_cdevpriv((void **)&info)) != 0) - return (error); - - if (cmd == KIOSETBUFSIZE) { - /* - * Set the size of the coverage buffer. Should be called - * before enabling coverage collection for that thread. - */ - if (info->state != KCOV_STATE_OPEN) { - return (EBUSY); - } - error = kcov_alloc(info, *(u_int *)data); - if (error == 0) - info->state = KCOV_STATE_READY; - return (error); - } - - mtx_lock_spin(&kcov_lock); - switch (cmd) { - case KIOENABLE: - if (info->state != KCOV_STATE_READY) { - error = EBUSY; - break; - } - if (td->td_kcov_info != NULL) { - error = EINVAL; - break; - } - mode = *(int *)data; - if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) { - error = EINVAL; - break; - } - KASSERT(info->thread == NULL, - ("Enabling kcov when already enabled")); - info->thread = td; - info->mode = mode; - /* - * Ensure the mode has been set before starting coverage - * tracing. - */ - atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING); - td->td_kcov_info = info; - break; - case KIODISABLE: - /* Only the currently enabled thread may disable itself */ - if (info->state != KCOV_STATE_RUNNING || - info != td->td_kcov_info) { - error = EINVAL; - break; - } - td->td_kcov_info = NULL; - atomic_store_int(&info->state, KCOV_STATE_READY); - /* *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
