From: Wei Yongjun <[email protected]> Currently the reporting of the percpu chunks leaking problem are not supported. This patch introduces this function.
Since __percpu pointer is not pointing directly to the actual chunks, this patch creates an object for __percpu pointer, but marks it as no scan block, only check whether this pointer is referenced by other blocks. Introduce two global variables, min_percpu_addr and max_percpu_addr, to store the range of valid percpu pointer values, in order to speed up pointer lookup when scanning blocks. Signed-off-by: Wei Yongjun <[email protected]> Signed-off-by: Chen Jun <[email protected]> --- mm/kmemleak.c | 71 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index c09c6b59eda6..feedb72f06f2 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -170,6 +170,8 @@ struct kmemleak_object { #define OBJECT_NO_SCAN (1 << 2) /* flag set to fully scan the object when scan_area allocation failed */ #define OBJECT_FULL_SCAN (1 << 3) +/* flag set to percpu ptr object */ +#define OBJECT_PERCPU (1 << 4) #define HEX_PREFIX " " /* number of bytes to print per line; must be 16 or 32 */ @@ -212,6 +214,9 @@ static int kmemleak_error; /* minimum and maximum address that may be valid pointers */ static unsigned long min_addr = ULONG_MAX; static unsigned long max_addr; +/* minimum and maximum address that may be valid percpu pointers */ +static unsigned long min_percpu_addr = ULONG_MAX; +static unsigned long max_percpu_addr; static struct task_struct *scan_thread; /* used to avoid reporting of recently allocated objects */ @@ -283,6 +288,9 @@ static void hex_dump_object(struct seq_file *seq, const u8 *ptr = (const u8 *)object->pointer; size_t len; + if (object->flags & OBJECT_PERCPU) + ptr = this_cpu_ptr((void __percpu *)object->pointer); + /* limit the number of lines to HEX_MAX_LINES */ len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); @@ -563,17 +571,32 @@ static int __save_stack_trace(unsigned long *trace) return stack_trace_save(trace, MAX_TRACE, 2); } +static void __update_address_range(struct kmemleak_object *object) +{ + unsigned long ptr = object->pointer; + size_t size = object->size; + unsigned long untagged_ptr; + + if (object->flags & OBJECT_PERCPU) { + min_percpu_addr = min(min_percpu_addr, ptr); + max_percpu_addr = max(max_percpu_addr, ptr + size); + } else { + untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); + min_addr = min(min_addr, untagged_ptr); + max_addr = max(max_addr, untagged_ptr + size); + } +} + /* * Create the metadata (struct kmemleak_object) corresponding to an allocated * memory block and add it to the object_list and object_tree_root. */ -static void create_object(unsigned long ptr, size_t size, int min_count, - gfp_t gfp) +static void __create_object(unsigned long ptr, size_t size, int min_count, + unsigned int obj_flags, gfp_t gfp) { unsigned long flags; struct kmemleak_object *object, *parent; struct rb_node **link, *rb_parent; - unsigned long untagged_ptr; object = mem_pool_alloc(gfp); if (!object) { @@ -587,7 +610,7 @@ static void create_object(unsigned long ptr, size_t size, int min_count, INIT_HLIST_HEAD(&object->area_list); raw_spin_lock_init(&object->lock); atomic_set(&object->use_count, 1); - object->flags = OBJECT_ALLOCATED; + object->flags = OBJECT_ALLOCATED | obj_flags; object->pointer = ptr; object->size = size; object->excess_ref = 0; @@ -619,9 +642,7 @@ static void create_object(unsigned long ptr, size_t size, int min_count, raw_spin_lock_irqsave(&kmemleak_lock, flags); - untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); - min_addr = min(min_addr, untagged_ptr); - max_addr = max(max_addr, untagged_ptr + size); + __update_address_range(object); link = &object_tree_root.rb_node; rb_parent = NULL; while (*link) { @@ -651,6 +672,19 @@ static void create_object(unsigned long ptr, size_t size, int min_count, raw_spin_unlock_irqrestore(&kmemleak_lock, flags); } +static void create_object(unsigned long ptr, size_t size, int min_count, + gfp_t gfp) +{ + __create_object(ptr, size, min_count, 0, gfp); +} + +static void create_object_percpu(unsigned long ptr, size_t size, int min_count, + gfp_t gfp) +{ + __create_object(ptr, size, min_count, OBJECT_PERCPU | OBJECT_NO_SCAN, + gfp); +} + /* * Mark the object as not allocated and schedule RCU freeing via put_object(). */ @@ -912,10 +946,12 @@ void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size, * Percpu allocations are only scanned and not reported as leaks * (min_count is set to 0). */ - if (kmemleak_enabled && ptr && !IS_ERR(ptr)) + if (kmemleak_enabled && ptr && !IS_ERR(ptr)) { for_each_possible_cpu(cpu) create_object((unsigned long)per_cpu_ptr(ptr, cpu), size, 0, gfp); + create_object_percpu((unsigned long)ptr, size, 1, gfp); + } } EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu); @@ -991,10 +1027,12 @@ void __ref kmemleak_free_percpu(const void __percpu *ptr) pr_debug("%s(0x%p)\n", __func__, ptr); - if (kmemleak_free_enabled && ptr && !IS_ERR(ptr)) + if (kmemleak_free_enabled && ptr && !IS_ERR(ptr)) { for_each_possible_cpu(cpu) delete_object_full((unsigned long)per_cpu_ptr(ptr, cpu)); + delete_object_full((unsigned long)ptr); + } } EXPORT_SYMBOL_GPL(kmemleak_free_percpu); @@ -1224,6 +1262,17 @@ static int scan_should_stop(void) return 0; } +static bool is_valid_address(unsigned long ptr) +{ + unsigned long untagged_ptr; + + if (ptr >= min_percpu_addr && ptr < max_percpu_addr) + return true; + + untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); + return (untagged_ptr >= min_addr && untagged_ptr < max_addr); +} + /* * Scan a memory block (exclusive range) for valid pointers and add those * found to the gray list. @@ -1235,7 +1284,6 @@ static void scan_block(void *_start, void *_end, unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER); unsigned long *end = _end - (BYTES_PER_POINTER - 1); unsigned long flags; - unsigned long untagged_ptr; raw_spin_lock_irqsave(&kmemleak_lock, flags); for (ptr = start; ptr < end; ptr++) { @@ -1250,8 +1298,7 @@ static void scan_block(void *_start, void *_end, pointer = *ptr; kasan_enable_current(); - untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer); - if (untagged_ptr < min_addr || untagged_ptr >= max_addr) + if (!is_valid_address(pointer)) continue; /* -- 2.25.0
