Add a selftest that exercises hmm_range_fault_unlockable() with a userfaultfd-backed mapping. The test:
1. Creates an anonymous mmap region 2. Registers it with userfaultfd (UFFDIO_REGISTER_MODE_MISSING) 3. Spawns a handler thread that responds to page faults by filling pages with a known pattern (0xAB) via UFFDIO_COPY 4. Issues HMM_DMIRROR_READ_UNLOCKABLE to the test_hmm driver, which calls hmm_range_fault_unlockable() internally 5. Verifies the device read back the data provided by the userfaultfd handler This requires changes to the test_hmm kernel module: - New dmirror_range_fault_unlockable() that uses the new HMM API - New dmirror_fault_unlockable() and dmirror_read_unlockable() wrappers - New HMM_DMIRROR_READ_UNLOCKABLE ioctl (0x09) Co-authored-by: Copilot <[email protected]> --- lib/test_hmm.c | 122 ++++++++++++++++++++++++++++++ lib/test_hmm_uapi.h | 1 tools/testing/selftests/mm/hmm-tests.c | 132 ++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+) diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 0964d53365e61..20b14e279a8bd 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -327,6 +327,84 @@ static int dmirror_range_fault(struct dmirror *dmirror, return ret; } +static int dmirror_range_fault_unlockable(struct dmirror *dmirror, + struct hmm_range *range) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long timeout = + jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + int locked; + int ret; + + while (true) { + if (time_after(jiffies, timeout)) { + ret = -EBUSY; + goto out; + } + + range->notifier_seq = mmu_interval_read_begin(range->notifier); + locked = 1; + mmap_read_lock(mm); + ret = hmm_range_fault_unlockable(range, &locked); + if (locked) + mmap_read_unlock(mm); + if (ret) { + if (ret == -EBUSY) + continue; + goto out; + } + if (!locked) + continue; + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + ret = dmirror_do_fault(dmirror, range); + + mutex_unlock(&dmirror->mutex); +out: + return ret; +} + +static int dmirror_fault_unlockable(struct dmirror *dmirror, + unsigned long start, + unsigned long end, bool write) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long addr; + unsigned long pfns[32]; + struct hmm_range range = { + .notifier = &dmirror->notifier, + .hmm_pfns = pfns, + .pfn_flags_mask = 0, + .default_flags = + HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + if (!mmget_not_zero(mm)) + return 0; + + for (addr = start; addr < end; addr = range.end) { + range.start = addr; + range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + + ret = dmirror_range_fault_unlockable(dmirror, &range); + if (ret) + break; + } + + mmput(mm); + return ret; +} + static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { @@ -426,6 +504,47 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) return ret; } +static int dmirror_read_unlockable(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault_unlockable(dmirror, start, end, false); + if (ret) + break; + cmd->faults++; + } + + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { @@ -1537,6 +1656,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, dmirror->flags = cmd.npages; ret = 0; break; + case HMM_DMIRROR_READ_UNLOCKABLE: + ret = dmirror_read_unlockable(dmirror, &cmd); + break; default: return -EINVAL; diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index f94c6d4573382..076df6df92275 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -38,6 +38,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd) #define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_READ_UNLOCKABLE _IOWR('H', 0x09, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0) diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index e8328c89d855e..12e988b96c158 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -26,6 +26,9 @@ #include <sys/mman.h> #include <sys/ioctl.h> #include <sys/time.h> +#include <sys/syscall.h> +#include <linux/userfaultfd.h> +#include <poll.h> /* @@ -2852,4 +2855,133 @@ TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) &thp_results, ®ular_results); } } +/* + * Test that HMM can fault in pages backed by userfaultfd using the + * hmm_range_fault_unlockable() path. This exercises the lock-drop retry + * logic in the HMM framework. + */ +struct uffd_thread_args { + int uffd; + void *page_buffer; + unsigned long page_size; +}; + +static void *uffd_handler_thread(void *arg) +{ + struct uffd_thread_args *args = arg; + struct uffd_msg msg; + struct uffdio_copy copy; + struct pollfd pollfd; + int ret; + + pollfd.fd = args->uffd; + pollfd.events = POLLIN; + + while (1) { + ret = poll(&pollfd, 1, 5000); + if (ret <= 0) + break; + + ret = read(args->uffd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) + break; + + if (msg.event != UFFD_EVENT_PAGEFAULT) + break; + + /* Fill the page with a known pattern */ + memset(args->page_buffer, 0xAB, args->page_size); + + copy.dst = msg.arg.pagefault.address & ~(args->page_size - 1); + copy.src = (unsigned long)args->page_buffer; + copy.len = args->page_size; + copy.mode = 0; + copy.copy = 0; + + ret = ioctl(args->uffd, UFFDIO_COPY, ©); + if (ret < 0) + break; + } + + return NULL; +} + +TEST_F(hmm, userfaultfd_read) +{ + struct hmm_buffer *buffer; + struct uffd_thread_args uffd_args; + unsigned long npages; + unsigned long size; + unsigned long i; + unsigned char *ptr; + pthread_t thread; + int uffd; + int ret; + struct uffdio_api api; + struct uffdio_register reg; + + npages = 4; + size = npages << self->page_shift; + + /* Create userfaultfd */ + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd < 0) + SKIP(return, "userfaultfd not available"); + + api.api = UFFD_API; + api.features = 0; + ret = ioctl(uffd, UFFDIO_API, &api); + ASSERT_EQ(ret, 0); + + buffer = malloc(sizeof(*buffer)); + ASSERT_NE(buffer, NULL); + + buffer->fd = -1; + buffer->size = size; + buffer->mirror = malloc(size); + ASSERT_NE(buffer->mirror, NULL); + + /* Create anonymous mapping */ + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ASSERT_NE(buffer->ptr, MAP_FAILED); + + /* Register the region with userfaultfd */ + reg.range.start = (unsigned long)buffer->ptr; + reg.range.len = size; + reg.mode = UFFDIO_REGISTER_MODE_MISSING; + ret = ioctl(uffd, UFFDIO_REGISTER, ®); + ASSERT_EQ(ret, 0); + + /* Set up the handler thread */ + uffd_args.uffd = uffd; + uffd_args.page_buffer = malloc(self->page_size); + ASSERT_NE(uffd_args.page_buffer, NULL); + uffd_args.page_size = self->page_size; + + ret = pthread_create(&thread, NULL, uffd_handler_thread, &uffd_args); + ASSERT_EQ(ret, 0); + + /* + * Use the unlockable read path which allows the mmap lock to be + * dropped during the fault, enabling userfaultfd resolution. + */ + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ_UNLOCKABLE, + buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + /* Verify the device read the data filled by the uffd handler */ + ptr = buffer->mirror; + for (i = 0; i < size; ++i) + ASSERT_EQ(ptr[i], (unsigned char)0xAB); + + pthread_join(thread, NULL); + free(uffd_args.page_buffer); + close(uffd); + hmm_buffer_free(buffer); +} + TEST_HARNESS_MAIN
