Move the page scanning logic out of read_page_owner() into a dedicated helper, in preparation for using it in the page_owner bpf_iter target.
The helper advances a struct page_owner_scan cursor to the next page eligible for page_owner output and returns true on a hit. On a hit it takes a snapshot of the page_owner record inside the page_ext RCU window. No change to the output is intended. handle is read from the snapshot, taken before the non-zero check, so the validated and reported handle are the same value. Signed-off-by: Zhen Ni <[email protected]> --- Changes in v2: - Route the zero-handle case through ext_put_continue. v1: https://lore.kernel.org/linux-mm/[email protected]/ --- mm/page_owner.c | 103 ++++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index fbbda7ba914b..56c75a669a6b 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -36,6 +36,16 @@ struct page_owner { pid_t free_tgid; }; +/* + * Cursor and per-page result of page_owner_next_eligible(). + */ +struct page_owner_scan { + /* resume point */ + unsigned long pfn; + struct page *page; + struct page_owner po_snap; +}; + struct stack { struct stack_record *stack_record; struct stack *next; @@ -725,37 +735,24 @@ void __dump_page_owner(const struct page *page) page_ext_put(page_ext); } -static ssize_t -read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) +/* + * Advance @scan to the next page eligible for page_owner output. + * On a hit, fill scan->pfn/page/po_snap and return true; the caller + * must advance scan->pfn past the hit before calling again to resume + * the scan. + */ +static bool page_owner_next_eligible(struct page_owner_scan *scan) { - unsigned long pfn; - struct page *page; - struct page_ext *page_ext; - struct page_owner *page_owner; - depot_stack_handle_t handle; - struct page_owner_filter_state *state = file->private_data; + unsigned long pfn = scan->pfn; - if (!static_branch_unlikely(&page_owner_inited)) - return -EINVAL; - - page = NULL; - if (*ppos == 0) - pfn = min_low_pfn; - else - pfn = *ppos; /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) pfn++; - /* Find an allocated page */ for (; pfn < max_pfn; pfn++) { - /* - * This temporary page_owner is required so - * that we can avoid the context switches while holding - * the rcu lock and copying the page owner information to - * user through copy_to_user() or GFP_KERNEL allocations. - */ - struct page_owner page_owner_tmp; + struct page_owner *page_owner; + struct page_ext *page_ext; + struct page *page; /* * If the new page is in a new MAX_ORDER_NR_PAGES area, @@ -798,37 +795,67 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) goto ext_put_continue; /* - * Access to page_ext->handle isn't synchronous so we should - * be careful to access it. + * Take a snapshot of the page_owner record inside the RCU + * window, then validate it, so that print_page_owner() + * reads the checked handle from the same snapshot. */ - handle = READ_ONCE(page_owner->handle); - if (!handle) + scan->pfn = pfn; + scan->page = page; + scan->po_snap = *page_owner; + + if (!scan->po_snap.handle) goto ext_put_continue; + page_ext_put(page_ext); + return true; + +ext_put_continue: + page_ext_put(page_ext); + cond_resched(); + } + + return false; +} + +static ssize_t +read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) +{ + struct page_owner_scan scan; + struct page_owner_filter_state *state = file->private_data; + + if (!static_branch_unlikely(&page_owner_inited)) + return -EINVAL; + + if (*ppos == 0) + scan.pfn = min_low_pfn; + else + scan.pfn = *ppos; + + /* Find an allocated page */ + while (page_owner_next_eligible(&scan)) { if (state->nid_filter_enabled) { int nid; - memdesc_flags_t page_flags = READ_ONCE(page->flags); + memdesc_flags_t page_flags = + READ_ONCE(scan.page->flags); /* * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid * VM_BUG_ON when accessing poisoned pages. */ if (page_flags.f == PAGE_POISON_PATTERN) - goto ext_put_continue; + goto skip_continue; nid = memdesc_nid(&page_flags); if (!node_isset(nid, state->nid_filter)) - goto ext_put_continue; + goto skip_continue; } /* Record the next PFN to read in the file offset */ - *ppos = pfn + 1; + *ppos = scan.pfn + 1; - page_owner_tmp = *page_owner; - page_ext_put(page_ext); - return print_page_owner(buf, count, pfn, page, - &page_owner_tmp, handle, state); -ext_put_continue: - page_ext_put(page_ext); + return print_page_owner(buf, count, scan.pfn, scan.page, + &scan.po_snap, scan.po_snap.handle, state); +skip_continue: + scan.pfn++; cond_resched(); } -- 2.20.1
