Add support to atomically turn the hardware watch on and off without allocation overhead.
The watch is pre-allocated and later retargeted. The current CPU is updated directly, while other CPUs are updated asynchronously via smp_call_function_single_async(). Signed-off-by: Jinchao Wang <[email protected]> --- mm/kstackwatch/kstackwatch.h | 2 + mm/kstackwatch/watch.c | 95 ++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/mm/kstackwatch/kstackwatch.h b/mm/kstackwatch/kstackwatch.h index 3ea191370970..2fa377843f17 100644 --- a/mm/kstackwatch/kstackwatch.h +++ b/mm/kstackwatch/kstackwatch.h @@ -41,5 +41,7 @@ const struct ksw_config *ksw_get_config(void); /* watch management */ int ksw_watch_init(void); void ksw_watch_exit(void); +int ksw_watch_on(u64 watch_addr, u64 watch_len); +void ksw_watch_off(void); #endif /* _KSTACKWATCH_H */ diff --git a/mm/kstackwatch/watch.c b/mm/kstackwatch/watch.c index d3399ac840b2..e02ffc3231ad 100644 --- a/mm/kstackwatch/watch.c +++ b/mm/kstackwatch/watch.c @@ -3,16 +3,23 @@ #include <linux/hw_breakpoint.h> #include <linux/perf_event.h> +#include <linux/preempt.h> #include <linux/printk.h> #include "kstackwatch.h" static struct perf_event *__percpu *watch_events; +static DEFINE_SPINLOCK(watch_lock); static unsigned long watch_holder; static struct perf_event_attr watch_attr; +static void ksw_watch_on_local_cpu(void *info); + +static DEFINE_PER_CPU(call_single_data_t, + watch_csd) = CSD_INIT(ksw_watch_on_local_cpu, NULL); + bool panic_on_catch; module_param(panic_on_catch, bool, 0644); MODULE_PARM_DESC(panic_on_catch, "panic immediately on corruption catch"); @@ -29,6 +36,94 @@ static void ksw_watch_handler(struct perf_event *bp, panic("Stack corruption detected"); } +static void ksw_watch_on_local_cpu(void *data) +{ + struct perf_event *bp; + int cpu; + int ret; + + preempt_disable(); + cpu = raw_smp_processor_id(); + bp = *per_cpu_ptr(watch_events, cpu); + if (!bp) { + preempt_enable(); + return; + } + + ret = modify_wide_hw_breakpoint_local(bp, &watch_attr); + preempt_enable(); + + if (ret) { + pr_err("failed to reinstall HWBP on CPU %d ret %d\n", cpu, + ret); + return; + } + + if (watch_attr.bp_addr == (unsigned long)&watch_holder) { + pr_debug("watch off CPU %d\n", cpu); + } else { + pr_debug("watch on CPU %d at 0x%llx (len %llu)\n", cpu, + watch_attr.bp_addr, watch_attr.bp_len); + } +} + +int ksw_watch_on(u64 watch_addr, u64 watch_len) +{ + unsigned long flags; + int cpu; + call_single_data_t *csd; + + if (!watch_addr) { + pr_err("watch with invalid address\n"); + return -EINVAL; + } + + spin_lock_irqsave(&watch_lock, flags); + + /* + * enforce singleton watch: + * - if a watch is already active (bp_addr != &watch_holder), + * - and not asking to reset it (watch_addr != &watch_holder) + * then reject with -EBUSY. + */ + if (watch_attr.bp_addr != (unsigned long)&watch_holder && + watch_addr != (unsigned long)&watch_holder) { + spin_unlock_irqrestore(&watch_lock, flags); + return -EBUSY; + } + + watch_attr.bp_addr = watch_addr; + watch_attr.bp_len = watch_len; + + /* ensure watchpoint update is visible to other CPUs before IPI */ + smp_wmb(); + + spin_unlock_irqrestore(&watch_lock, flags); + + if (watch_addr == (unsigned long)&watch_holder) + pr_debug("watch off starting\n"); + else + pr_debug("watch on starting\n"); + + cpus_read_lock(); + for_each_online_cpu(cpu) { + if (cpu == raw_smp_processor_id()) { + ksw_watch_on_local_cpu(NULL); + } else { + csd = &per_cpu(watch_csd, cpu); + smp_call_function_single_async(cpu, csd); + } + } + cpus_read_unlock(); + + return 0; +} + +void ksw_watch_off(void) +{ + ksw_watch_on((unsigned long)&watch_holder, sizeof(watch_holder)); +} + int ksw_watch_init(void) { int ret; -- 2.43.0
