Hi Andrea, On 5 March 2015 at 18:17, Andrea Arcangeli <aarca...@redhat.com> wrote: > Once an userfaultfd has been created and certain region of the process > virtual address space have been registered into it, the thread > responsible for doing the memory externalization can manage the page > faults in userland by talking to the kernel using the userfaultfd > protocol.
Is there someting like a man page for this new syscall? Thanks, Michael > poll() can be used to know when there are new pending userfaults to be > read (POLLIN). > > Signed-off-by: Andrea Arcangeli <aarca...@redhat.com> > --- > fs/userfaultfd.c | 977 > +++++++++++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 977 insertions(+) > create mode 100644 fs/userfaultfd.c > > diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c > new file mode 100644 > index 0000000..6b31967 > --- /dev/null > +++ b/fs/userfaultfd.c > @@ -0,0 +1,977 @@ > +/* > + * fs/userfaultfd.c > + * > + * Copyright (C) 2007 Davide Libenzi <davi...@xmailserver.org> > + * Copyright (C) 2008-2009 Red Hat, Inc. > + * Copyright (C) 2015 Red Hat, Inc. > + * > + * This work is licensed under the terms of the GNU GPL, version 2. See > + * the COPYING file in the top-level directory. > + * > + * Some part derived from fs/eventfd.c (anon inode setup) and > + * mm/ksm.c (mm hashing). > + */ > + > +#include <linux/hashtable.h> > +#include <linux/sched.h> > +#include <linux/mm.h> > +#include <linux/poll.h> > +#include <linux/slab.h> > +#include <linux/seq_file.h> > +#include <linux/file.h> > +#include <linux/bug.h> > +#include <linux/anon_inodes.h> > +#include <linux/syscalls.h> > +#include <linux/userfaultfd_k.h> > +#include <linux/mempolicy.h> > +#include <linux/ioctl.h> > +#include <linux/security.h> > + > +enum userfaultfd_state { > + UFFD_STATE_WAIT_API, > + UFFD_STATE_RUNNING, > +}; > + > +struct userfaultfd_ctx { > + /* pseudo fd refcounting */ > + atomic_t refcount; > + /* waitqueue head for the userfaultfd page faults */ > + wait_queue_head_t fault_wqh; > + /* waitqueue head for the pseudo fd to wakeup poll/read */ > + wait_queue_head_t fd_wqh; > + /* userfaultfd syscall flags */ > + unsigned int flags; > + /* state machine */ > + enum userfaultfd_state state; > + /* released */ > + bool released; > + /* mm with one ore more vmas attached to this userfaultfd_ctx */ > + struct mm_struct *mm; > +}; > + > +struct userfaultfd_wait_queue { > + unsigned long address; > + wait_queue_t wq; > + bool pending; > + struct userfaultfd_ctx *ctx; > +}; > + > +struct userfaultfd_wake_range { > + unsigned long start; > + unsigned long len; > +}; > + > +static int userfaultfd_wake_function(wait_queue_t *wq, unsigned mode, > + int wake_flags, void *key) > +{ > + struct userfaultfd_wake_range *range = key; > + int ret; > + struct userfaultfd_wait_queue *uwq; > + unsigned long start, len; > + > + uwq = container_of(wq, struct userfaultfd_wait_queue, wq); > + ret = 0; > + /* don't wake the pending ones to avoid reads to block */ > + if (uwq->pending && !ACCESS_ONCE(uwq->ctx->released)) > + goto out; > + /* len == 0 means wake all */ > + start = range->start; > + len = range->len; > + if (len && (start > uwq->address || start + len <= uwq->address)) > + goto out; > + ret = wake_up_state(wq->private, mode); > + if (ret) > + /* wake only once, autoremove behavior */ > + list_del_init(&wq->task_list); > +out: > + return ret; > +} > + > +/** > + * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd > + * context. > + * @ctx: [in] Pointer to the userfaultfd context. > + * > + * Returns: In case of success, returns not zero. > + */ > +static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx) > +{ > + if (!atomic_inc_not_zero(&ctx->refcount)) > + BUG(); > +} > + > +/** > + * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd > + * context. > + * @ctx: [in] Pointer to userfaultfd context. > + * > + * The userfaultfd context reference must have been previously acquired > either > + * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget(). > + */ > +static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) > +{ > + if (atomic_dec_and_test(&ctx->refcount)) { > + mmdrop(ctx->mm); > + kfree(ctx); > + } > +} > + > +static inline unsigned long userfault_address(unsigned long address, > + unsigned int flags, > + unsigned long reason) > +{ > + BUILD_BUG_ON(PAGE_SHIFT < UFFD_BITS); > + address &= PAGE_MASK; > + if (flags & FAULT_FLAG_WRITE) > + /* > + * Encode "write" fault information in the LSB of the > + * address read by userland, without depending on > + * FAULT_FLAG_WRITE kernel internal value. > + */ > + address |= UFFD_BIT_WRITE; > + if (reason & VM_UFFD_WP) > + /* > + * Encode "reason" fault information as bit number 1 > + * in the address read by userland. If bit number 1 is > + * clear it means the reason is a VM_FAULT_MISSING > + * fault. > + */ > + address |= UFFD_BIT_WP; > + return address; > +} > + > +/* > + * The locking rules involved in returning VM_FAULT_RETRY depending on > + * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and > + * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" > + * recommendation in __lock_page_or_retry is not an understatement. > + * > + * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released > + * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is > + * not set. > + * > + * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not > + * set, VM_FAULT_RETRY can still be returned if and only if there are > + * fatal_signal_pending()s, and the mmap_sem must be released before > + * returning it. > + */ > +int handle_userfault(struct vm_area_struct *vma, unsigned long address, > + unsigned int flags, unsigned long reason) > +{ > + struct mm_struct *mm = vma->vm_mm; > + struct userfaultfd_ctx *ctx; > + struct userfaultfd_wait_queue uwq; > + > + BUG_ON(!rwsem_is_locked(&mm->mmap_sem)); > + > + ctx = vma->vm_userfaultfd_ctx.ctx; > + if (!ctx) > + return VM_FAULT_SIGBUS; > + > + BUG_ON(ctx->mm != mm); > + > + VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP)); > + VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP)); > + > + /* > + * If it's already released don't get it. This avoids to loop > + * in __get_user_pages if userfaultfd_release waits on the > + * caller of handle_userfault to release the mmap_sem. > + */ > + if (unlikely(ACCESS_ONCE(ctx->released))) > + return VM_FAULT_SIGBUS; > + > + /* check that we can return VM_FAULT_RETRY */ > + if (unlikely(!(flags & FAULT_FLAG_ALLOW_RETRY))) { > + /* > + * Validate the invariant that nowait must allow retry > + * to be sure not to return SIGBUS erroneously on > + * nowait invocations. > + */ > + BUG_ON(flags & FAULT_FLAG_RETRY_NOWAIT); > +#ifdef CONFIG_DEBUG_VM > + if (printk_ratelimit()) { > + printk(KERN_WARNING > + "FAULT_FLAG_ALLOW_RETRY missing %x\n", flags); > + dump_stack(); > + } > +#endif > + return VM_FAULT_SIGBUS; > + } > + > + /* > + * Handle nowait, not much to do other than tell it to retry > + * and wait. > + */ > + if (flags & FAULT_FLAG_RETRY_NOWAIT) > + return VM_FAULT_RETRY; > + > + /* take the reference before dropping the mmap_sem */ > + userfaultfd_ctx_get(ctx); > + > + /* be gentle and immediately relinquish the mmap_sem */ > + up_read(&mm->mmap_sem); > + > + init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function); > + uwq.wq.private = current; > + uwq.address = userfault_address(address, flags, reason); > + uwq.pending = true; > + uwq.ctx = ctx; > + > + spin_lock(&ctx->fault_wqh.lock); > + /* > + * After the __add_wait_queue the uwq is visible to userland > + * through poll/read(). > + */ > + __add_wait_queue(&ctx->fault_wqh, &uwq.wq); > + for (;;) { > + set_current_state(TASK_KILLABLE); > + if (!uwq.pending || ACCESS_ONCE(ctx->released) || > + fatal_signal_pending(current)) > + break; > + spin_unlock(&ctx->fault_wqh.lock); > + > + wake_up_poll(&ctx->fd_wqh, POLLIN); > + schedule(); > + > + spin_lock(&ctx->fault_wqh.lock); > + } > + __remove_wait_queue(&ctx->fault_wqh, &uwq.wq); > + __set_current_state(TASK_RUNNING); > + spin_unlock(&ctx->fault_wqh.lock); > + > + /* > + * ctx may go away after this if the userfault pseudo fd is > + * already released. > + */ > + userfaultfd_ctx_put(ctx); > + > + return VM_FAULT_RETRY; > +} > + > +static int userfaultfd_release(struct inode *inode, struct file *file) > +{ > + struct userfaultfd_ctx *ctx = file->private_data; > + struct mm_struct *mm = ctx->mm; > + struct vm_area_struct *vma, *prev; > + /* len == 0 means wake all */ > + struct userfaultfd_wake_range range = { .len = 0, }; > + unsigned long new_flags; > + > + ACCESS_ONCE(ctx->released) = true; > + > + /* > + * Flush page faults out of all CPUs. NOTE: all page faults > + * must be retried without returning VM_FAULT_SIGBUS if > + * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx > + * changes while handle_userfault released the mmap_sem. So > + * it's critical that released is set to true (above), before > + * taking the mmap_sem for writing. > + */ > + down_write(&mm->mmap_sem); > + prev = NULL; > + for (vma = mm->mmap; vma; vma = vma->vm_next) { > + cond_resched(); > + BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ > + !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); > + if (vma->vm_userfaultfd_ctx.ctx != ctx) { > + prev = vma; > + continue; > + } > + new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); > + prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, > + new_flags, vma->anon_vma, > + vma->vm_file, vma->vm_pgoff, > + vma_policy(vma), > + NULL_VM_UFFD_CTX); > + if (prev) > + vma = prev; > + else > + prev = vma; > + vma->vm_flags = new_flags; > + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; > + } > + up_write(&mm->mmap_sem); > + > + /* > + * After no new page faults can wait on this fault_wqh, flush > + * the last page faults that may have been already waiting on > + * the fault_wqh. > + */ > + spin_lock(&ctx->fault_wqh.lock); > + __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, &range); > + spin_unlock(&ctx->fault_wqh.lock); > + > + wake_up_poll(&ctx->fd_wqh, POLLHUP); > + userfaultfd_ctx_put(ctx); > + return 0; > +} > + > +static inline unsigned int find_userfault(struct userfaultfd_ctx *ctx, > + struct userfaultfd_wait_queue **uwq) > +{ > + wait_queue_t *wq; > + struct userfaultfd_wait_queue *_uwq; > + unsigned int ret = 0; > + > + spin_lock(&ctx->fault_wqh.lock); > + list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) { > + _uwq = container_of(wq, struct userfaultfd_wait_queue, wq); > + if (_uwq->pending) { > + ret = POLLIN; > + if (uwq) > + *uwq = _uwq; > + break; > + } > + } > + spin_unlock(&ctx->fault_wqh.lock); > + > + return ret; > +} > + > +static unsigned int userfaultfd_poll(struct file *file, poll_table *wait) > +{ > + struct userfaultfd_ctx *ctx = file->private_data; > + > + poll_wait(file, &ctx->fd_wqh, wait); > + > + switch (ctx->state) { > + case UFFD_STATE_WAIT_API: > + return POLLERR; > + case UFFD_STATE_RUNNING: > + return find_userfault(ctx, NULL); > + default: > + BUG(); > + } > +} > + > +static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, > + __u64 *addr) > +{ > + ssize_t ret; > + DECLARE_WAITQUEUE(wait, current); > + struct userfaultfd_wait_queue *uwq = NULL; > + > + /* always take the fd_wqh lock before the fault_wqh lock */ > + spin_lock(&ctx->fd_wqh.lock); > + __add_wait_queue(&ctx->fd_wqh, &wait); > + for (;;) { > + set_current_state(TASK_INTERRUPTIBLE); > + if (find_userfault(ctx, &uwq)) { > + uwq->pending = false; > + /* careful to always initialize addr if ret == 0 */ > + *addr = uwq->address; > + ret = 0; > + break; > + } > + if (signal_pending(current)) { > + ret = -ERESTARTSYS; > + break; > + } > + if (no_wait) { > + ret = -EAGAIN; > + break; > + } > + spin_unlock(&ctx->fd_wqh.lock); > + schedule(); > + spin_lock_irq(&ctx->fd_wqh.lock); > + } > + __remove_wait_queue(&ctx->fd_wqh, &wait); > + __set_current_state(TASK_RUNNING); > + spin_unlock_irq(&ctx->fd_wqh.lock); > + > + return ret; > +} > + > +static ssize_t userfaultfd_read(struct file *file, char __user *buf, > + size_t count, loff_t *ppos) > +{ > + struct userfaultfd_ctx *ctx = file->private_data; > + ssize_t _ret, ret = 0; > + /* careful to always initialize addr if ret == 0 */ > + __u64 uninitialized_var(addr); > + int no_wait = file->f_flags & O_NONBLOCK; > + > + if (ctx->state == UFFD_STATE_WAIT_API) > + return -EINVAL; > + BUG_ON(ctx->state != UFFD_STATE_RUNNING); > + > + for (;;) { > + if (count < sizeof(addr)) > + return ret ? ret : -EINVAL; > + _ret = userfaultfd_ctx_read(ctx, no_wait, &addr); > + if (_ret < 0) > + return ret ? ret : _ret; > + if (put_user(addr, (__u64 __user *) buf)) > + return ret ? ret : -EFAULT; > + ret += sizeof(addr); > + buf += sizeof(addr); > + count -= sizeof(addr); > + /* > + * Allow to read more than one fault at time but only > + * block if waiting for the very first one. > + */ > + no_wait = O_NONBLOCK; > + } > +} > + > +static int __wake_userfault(struct userfaultfd_ctx *ctx, > + struct userfaultfd_wake_range *range) > +{ > + wait_queue_t *wq; > + struct userfaultfd_wait_queue *uwq; > + int ret; > + unsigned long start, end; > + > + start = range->start; > + end = range->start + range->len; > + > + ret = -ENOENT; > + spin_lock(&ctx->fault_wqh.lock); > + list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) { > + uwq = container_of(wq, struct userfaultfd_wait_queue, wq); > + if (uwq->pending) > + continue; > + if (uwq->address >= start && uwq->address < end) { > + ret = 0; > + /* wake all in the range and autoremove */ > + __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, > + range); > + break; > + } > + } > + spin_unlock(&ctx->fault_wqh.lock); > + > + return ret; > +} > + > +static __always_inline int wake_userfault(struct userfaultfd_ctx *ctx, > + struct userfaultfd_wake_range > *range) > +{ > + if (!waitqueue_active(&ctx->fault_wqh)) > + return -ENOENT; > + > + return __wake_userfault(ctx, range); > +} > + > +static __always_inline int validate_range(struct mm_struct *mm, > + __u64 start, __u64 len) > +{ > + __u64 task_size = mm->task_size; > + > + if (start & ~PAGE_MASK) > + return -EINVAL; > + if (len & ~PAGE_MASK) > + return -EINVAL; > + if (!len) > + return -EINVAL; > + if (start < mmap_min_addr) > + return -EINVAL; > + if (start >= task_size) > + return -EINVAL; > + if (len > task_size - start) > + return -EINVAL; > + return 0; > +} > + > +static int userfaultfd_register(struct userfaultfd_ctx *ctx, > + unsigned long arg) > +{ > + struct mm_struct *mm = ctx->mm; > + struct vm_area_struct *vma, *prev, *cur; > + int ret; > + struct uffdio_register uffdio_register; > + struct uffdio_register __user *user_uffdio_register; > + unsigned long vm_flags, new_flags; > + bool found; > + unsigned long start, end, vma_end; > + > + user_uffdio_register = (struct uffdio_register __user *) arg; > + > + ret = -EFAULT; > + if (copy_from_user(&uffdio_register, user_uffdio_register, > + sizeof(uffdio_register)-sizeof(__u64))) > + goto out; > + > + ret = -EINVAL; > + if (!uffdio_register.mode) > + goto out; > + if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING| > + UFFDIO_REGISTER_MODE_WP)) > + goto out; > + vm_flags = 0; > + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) > + vm_flags |= VM_UFFD_MISSING; > + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { > + vm_flags |= VM_UFFD_WP; > + /* > + * FIXME: remove the below error constraint by > + * implementing the wprotect tracking mode. > + */ > + ret = -EINVAL; > + goto out; > + } > + > + ret = validate_range(mm, uffdio_register.range.start, > + uffdio_register.range.len); > + if (ret) > + goto out; > + > + start = uffdio_register.range.start; > + end = start + uffdio_register.range.len; > + > + down_write(&mm->mmap_sem); > + vma = find_vma_prev(mm, start, &prev); > + > + ret = -ENOMEM; > + if (!vma) > + goto out_unlock; > + > + /* check that there's at least one vma in the range */ > + ret = -EINVAL; > + if (vma->vm_start >= end) > + goto out_unlock; > + > + /* > + * Search for not compatible vmas. > + * > + * FIXME: this shall be relaxed later so that it doesn't fail > + * on tmpfs backed vmas (in addition to the current allowance > + * on anonymous vmas). > + */ > + found = false; > + for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { > + cond_resched(); > + > + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ > + !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); > + > + /* check not compatible vmas */ > + ret = -EINVAL; > + if (cur->vm_ops) > + goto out_unlock; > + > + /* > + * Check that this vma isn't already owned by a > + * different userfaultfd. We can't allow more than one > + * userfaultfd to own a single vma simultaneously or we > + * wouldn't know which one to deliver the userfaults to. > + */ > + ret = -EBUSY; > + if (cur->vm_userfaultfd_ctx.ctx && > + cur->vm_userfaultfd_ctx.ctx != ctx) > + goto out_unlock; > + > + found = true; > + } > + BUG_ON(!found); > + > + /* > + * Now that we scanned all vmas we can already tell userland which > + * ioctls methods are guaranteed to succeed on this range. > + */ > + ret = -EFAULT; > + if (put_user(UFFD_API_RANGE_IOCTLS, &user_uffdio_register->ioctls)) > + goto out_unlock; > + > + if (vma->vm_start < start) > + prev = vma; > + > + ret = 0; > + do { > + cond_resched(); > + > + BUG_ON(vma->vm_ops); > + BUG_ON(vma->vm_userfaultfd_ctx.ctx && > + vma->vm_userfaultfd_ctx.ctx != ctx); > + > + /* > + * Nothing to do: this vma is already registered into this > + * userfaultfd and with the right tracking mode too. > + */ > + if (vma->vm_userfaultfd_ctx.ctx == ctx && > + (vma->vm_flags & vm_flags) == vm_flags) > + goto skip; > + > + if (vma->vm_start > start) > + start = vma->vm_start; > + vma_end = min(end, vma->vm_end); > + > + new_flags = (vma->vm_flags & ~vm_flags) | vm_flags; > + prev = vma_merge(mm, prev, start, vma_end, new_flags, > + vma->anon_vma, vma->vm_file, vma->vm_pgoff, > + vma_policy(vma), > + ((struct vm_userfaultfd_ctx){ ctx })); > + if (prev) { > + vma = prev; > + goto next; > + } > + if (vma->vm_start < start) { > + ret = split_vma(mm, vma, start, 1); > + if (ret) > + break; > + } > + if (vma->vm_end > end) { > + ret = split_vma(mm, vma, end, 0); > + if (ret) > + break; > + } > + next: > + /* > + * In the vma_merge() successful mprotect-like case 8: > + * the next vma was merged into the current one and > + * the current one has not been updated yet. > + */ > + vma->vm_flags = new_flags; > + vma->vm_userfaultfd_ctx.ctx = ctx; > + > + skip: > + prev = vma; > + start = vma->vm_end; > + vma = vma->vm_next; > + } while (vma && vma->vm_start < end); > +out_unlock: > + up_write(&mm->mmap_sem); > +out: > + return ret; > +} > + > +static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, > + unsigned long arg) > +{ > + struct mm_struct *mm = ctx->mm; > + struct vm_area_struct *vma, *prev, *cur; > + int ret; > + struct uffdio_range uffdio_unregister; > + unsigned long new_flags; > + bool found; > + unsigned long start, end, vma_end; > + const void __user *buf = (void __user *)arg; > + > + ret = -EFAULT; > + if (copy_from_user(&uffdio_unregister, buf, > sizeof(uffdio_unregister))) > + goto out; > + > + ret = validate_range(mm, uffdio_unregister.start, > + uffdio_unregister.len); > + if (ret) > + goto out; > + > + start = uffdio_unregister.start; > + end = start + uffdio_unregister.len; > + > + down_write(&mm->mmap_sem); > + vma = find_vma_prev(mm, start, &prev); > + > + ret = -ENOMEM; > + if (!vma) > + goto out_unlock; > + > + /* check that there's at least one vma in the range */ > + ret = -EINVAL; > + if (vma->vm_start >= end) > + goto out_unlock; > + > + /* > + * Search for not compatible vmas. > + * > + * FIXME: this shall be relaxed later so that it doesn't fail > + * on tmpfs backed vmas (in addition to the current allowance > + * on anonymous vmas). > + */ > + found = false; > + ret = -EINVAL; > + for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { > + cond_resched(); > + > + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ > + !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); > + > + /* > + * Check not compatible vmas, not strictly required > + * here as not compatible vmas cannot have an > + * userfaultfd_ctx registered on them, but this > + * provides for more strict behavior to notice > + * unregistration errors. > + */ > + if (cur->vm_ops) > + goto out_unlock; > + > + found = true; > + } > + BUG_ON(!found); > + > + if (vma->vm_start < start) > + prev = vma; > + > + ret = 0; > + do { > + cond_resched(); > + > + BUG_ON(vma->vm_ops); > + > + /* > + * Nothing to do: this vma is already registered into this > + * userfaultfd and with the right tracking mode too. > + */ > + if (!vma->vm_userfaultfd_ctx.ctx) > + goto skip; > + > + if (vma->vm_start > start) > + start = vma->vm_start; > + vma_end = min(end, vma->vm_end); > + > + new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); > + prev = vma_merge(mm, prev, start, vma_end, new_flags, > + vma->anon_vma, vma->vm_file, vma->vm_pgoff, > + vma_policy(vma), > + NULL_VM_UFFD_CTX); > + if (prev) { > + vma = prev; > + goto next; > + } > + if (vma->vm_start < start) { > + ret = split_vma(mm, vma, start, 1); > + if (ret) > + break; > + } > + if (vma->vm_end > end) { > + ret = split_vma(mm, vma, end, 0); > + if (ret) > + break; > + } > + next: > + /* > + * In the vma_merge() successful mprotect-like case 8: > + * the next vma was merged into the current one and > + * the current one has not been updated yet. > + */ > + vma->vm_flags = new_flags; > + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; > + > + skip: > + prev = vma; > + start = vma->vm_end; > + vma = vma->vm_next; > + } while (vma && vma->vm_start < end); > +out_unlock: > + up_write(&mm->mmap_sem); > +out: > + return ret; > +} > + > +/* > + * This is mostly needed to re-wakeup those userfaults that were still > + * pending when userland wake them up the first time. We don't wake > + * the pending one to avoid blocking reads to block, or non blocking > + * read to return -EAGAIN, if used with POLLIN, to avoid userland > + * doubts on why POLLIN wasn't reliable. > + */ > +static int userfaultfd_wake(struct userfaultfd_ctx *ctx, > + unsigned long arg) > +{ > + int ret; > + struct uffdio_range uffdio_wake; > + struct userfaultfd_wake_range range; > + const void __user *buf = (void __user *)arg; > + > + ret = -EFAULT; > + if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake))) > + goto out; > + > + ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len); > + if (ret) > + goto out; > + > + range.start = uffdio_wake.start; > + range.len = uffdio_wake.len; > + > + /* > + * len == 0 means wake all and we don't want to wake all here, > + * so check it again to be sure. > + */ > + VM_BUG_ON(!range.len); > + > + ret = wake_userfault(ctx, &range); > + > +out: > + return ret; > +} > + > +/* > + * userland asks for a certain API version and we return which bits > + * and ioctl commands are implemented in this kernel for such API > + * version or -EINVAL if unknown. > + */ > +static int userfaultfd_api(struct userfaultfd_ctx *ctx, > + unsigned long arg) > +{ > + struct uffdio_api uffdio_api; > + void __user *buf = (void __user *)arg; > + int ret; > + > + ret = -EINVAL; > + if (ctx->state != UFFD_STATE_WAIT_API) > + goto out; > + ret = -EFAULT; > + if (copy_from_user(&uffdio_api, buf, sizeof(__u64))) > + goto out; > + if (uffdio_api.api != UFFD_API) { > + /* careful not to leak info, we only read the first 8 bytes */ > + memset(&uffdio_api, 0, sizeof(uffdio_api)); > + if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) > + goto out; > + ret = -EINVAL; > + goto out; > + } > + /* careful not to leak info, we only read the first 8 bytes */ > + uffdio_api.bits = UFFD_API_BITS; > + uffdio_api.ioctls = UFFD_API_IOCTLS; > + ret = -EFAULT; > + if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) > + goto out; > + ctx->state = UFFD_STATE_RUNNING; > + ret = 0; > +out: > + return ret; > +} > + > +static long userfaultfd_ioctl(struct file *file, unsigned cmd, > + unsigned long arg) > +{ > + int ret = -EINVAL; > + struct userfaultfd_ctx *ctx = file->private_data; > + > + switch(cmd) { > + case UFFDIO_API: > + ret = userfaultfd_api(ctx, arg); > + break; > + case UFFDIO_REGISTER: > + ret = userfaultfd_register(ctx, arg); > + break; > + case UFFDIO_UNREGISTER: > + ret = userfaultfd_unregister(ctx, arg); > + break; > + case UFFDIO_WAKE: > + ret = userfaultfd_wake(ctx, arg); > + break; > + } > + return ret; > +} > + > +#ifdef CONFIG_PROC_FS > +static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) > +{ > + struct userfaultfd_ctx *ctx = f->private_data; > + wait_queue_t *wq; > + struct userfaultfd_wait_queue *uwq; > + unsigned long pending = 0, total = 0; > + > + spin_lock(&ctx->fault_wqh.lock); > + list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) { > + uwq = container_of(wq, struct userfaultfd_wait_queue, wq); > + if (uwq->pending) > + pending++; > + total++; > + } > + spin_unlock(&ctx->fault_wqh.lock); > + > + /* > + * If more protocols will be added, there will be all shown > + * separated by a space. Like this: > + * protocols: 0xaa 0xbb > + */ > + seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", > + pending, total, UFFD_API, UFFD_API_BITS, > + UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); > +} > +#endif > + > +static const struct file_operations userfaultfd_fops = { > +#ifdef CONFIG_PROC_FS > + .show_fdinfo = userfaultfd_show_fdinfo, > +#endif > + .release = userfaultfd_release, > + .poll = userfaultfd_poll, > + .read = userfaultfd_read, > + .unlocked_ioctl = userfaultfd_ioctl, > + .compat_ioctl = userfaultfd_ioctl, > + .llseek = noop_llseek, > +}; > + > +/** > + * userfaultfd_file_create - Creates an userfaultfd file pointer. > + * @flags: Flags for the userfaultfd file. > + * > + * This function creates an userfaultfd file pointer, w/out installing > + * it into the fd table. This is useful when the userfaultfd file is > + * used during the initialization of data structures that require > + * extra setup after the userfaultfd creation. So the userfaultfd > + * creation is split into the file pointer creation phase, and the > + * file descriptor installation phase. In this way races with > + * userspace closing the newly installed file descriptor can be > + * avoided. Returns an userfaultfd file pointer, or a proper error > + * pointer. > + */ > +static struct file *userfaultfd_file_create(int flags) > +{ > + struct file *file; > + struct userfaultfd_ctx *ctx; > + > + BUG_ON(!current->mm); > + > + /* Check the UFFD_* constants for consistency. */ > + BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC); > + BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK); > + > + file = ERR_PTR(-EINVAL); > + if (flags & ~UFFD_SHARED_FCNTL_FLAGS) > + goto out; > + > + file = ERR_PTR(-ENOMEM); > + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); > + if (!ctx) > + goto out; > + > + atomic_set(&ctx->refcount, 1); > + init_waitqueue_head(&ctx->fault_wqh); > + init_waitqueue_head(&ctx->fd_wqh); > + ctx->flags = flags; > + ctx->state = UFFD_STATE_WAIT_API; > + ctx->released = false; > + ctx->mm = current->mm; > + /* prevent the mm struct to be freed */ > + atomic_inc(&ctx->mm->mm_count); > + > + file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx, > + O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS)); > + if (IS_ERR(file)) > + kfree(ctx); > +out: > + return file; > +} > + > +SYSCALL_DEFINE1(userfaultfd, int, flags) > +{ > + int fd, error; > + struct file *file; > + > + error = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS); > + if (error < 0) > + return error; > + fd = error; > + > + file = userfaultfd_file_create(flags); > + if (IS_ERR(file)) { > + error = PTR_ERR(file); > + goto err_put_unused_fd; > + } > + fd_install(fd, file); > + > + return fd; > + > +err_put_unused_fd: > + put_unused_fd(fd); > + > + return error; > +} > -- > To unsubscribe from this list: send the line "unsubscribe linux-api" in > the body of a message to majord...@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html -- Michael Kerrisk Linux man-pages maintainer; http://www.kernel.org/doc/man-pages/ Linux/UNIX System Programming Training: http://man7.org/training/ -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html