Author: markj Date: Thu Dec 12 21:13:20 2019 New Revision: 355672 URL: https://svnweb.freebsd.org/changeset/base/355672
Log: Implement atomic state updates using the new vm_page_astate_t structure. Introduce primitives vm_page_astate_load() and vm_page_astate_fcmpset() to operate on the 32-bit per-page atomic state. Modify vm_page_pqstate_fcmpset() to use them. No functional change intended. Introduce PGA_QUEUE_OP_MASK, a subset of PGA_QUEUE_STATE_MASK that only includes queue operation flags. This will be used in subsequent patches. Reviewed by: alc, jeff, kib Sponsored by: Netflix, Intel Differential Revision: https://reviews.freebsd.org/D22753 Modified: head/sys/vm/vm_page.h Modified: head/sys/vm/vm_page.h ============================================================================== --- head/sys/vm/vm_page.h Thu Dec 12 20:55:43 2019 (r355671) +++ head/sys/vm/vm_page.h Thu Dec 12 21:13:20 2019 (r355672) @@ -439,8 +439,8 @@ extern struct mtx_padalign pa_lock[]; #define PGA_REQUEUE_HEAD 0x0040 /* page requeue should bypass LRU */ #define PGA_NOSYNC 0x0080 /* do not collect for syncer */ -#define PGA_QUEUE_STATE_MASK (PGA_ENQUEUED | PGA_DEQUEUE | PGA_REQUEUE | \ - PGA_REQUEUE_HEAD) +#define PGA_QUEUE_OP_MASK (PGA_DEQUEUE | PGA_REQUEUE | PGA_REQUEUE_HEAD) +#define PGA_QUEUE_STATE_MASK (PGA_ENQUEUED | PGA_QUEUE_OP_MASK) /* * Page flags. If changed at any other time than page allocation or @@ -756,36 +756,37 @@ void vm_page_assert_pga_writeable(vm_page_t m, uint16_ #define VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits) (void)0 #endif +#define VM_PAGE_AFLAG_SHIFT (__offsetof(vm_page_astate_t, flags) * NBBY) + /* - * We want to use atomic updates for the aflags field, which is 8 bits wide. - * However, not all architectures support atomic operations on 8-bit - * destinations. In order that we can easily use a 32-bit operation, we - * require that the aflags field be 32-bit aligned. + * Load a snapshot of a page's 32-bit atomic state. */ -_Static_assert(offsetof(struct vm_page, a.flags) % sizeof(uint32_t) == 0, - "aflags field is not 32-bit aligned"); +static inline vm_page_astate_t +vm_page_astate_load(vm_page_t m) +{ + vm_page_astate_t a; + a._bits = atomic_load_32(&m->a); + return (a); +} + /* - * We want to be able to update the aflags and queue fields atomically in - * the same operation. + * Atomically compare and set a page's atomic state. */ -_Static_assert(offsetof(struct vm_page, a.flags) / sizeof(uint32_t) == - offsetof(struct vm_page, a.queue) / sizeof(uint32_t), - "aflags and queue fields do not belong to the same 32-bit word"); -_Static_assert(offsetof(struct vm_page, a.queue) % sizeof(uint32_t) == 2, - "queue field is at an unexpected offset"); -_Static_assert(sizeof(((struct vm_page *)NULL)->a.queue) == 1, - "queue field has an unexpected size"); +static inline bool +vm_page_astate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new) +{ -#if BYTE_ORDER == LITTLE_ENDIAN -#define VM_PAGE_AFLAG_SHIFT 0 -#define VM_PAGE_QUEUE_SHIFT 16 -#else -#define VM_PAGE_AFLAG_SHIFT 16 -#define VM_PAGE_QUEUE_SHIFT 8 -#endif -#define VM_PAGE_QUEUE_MASK (0xff << VM_PAGE_QUEUE_SHIFT) + KASSERT(new.queue == PQ_INACTIVE || (new.flags & PGA_REQUEUE_HEAD) == 0, + ("%s: invalid head requeue request for page %p", __func__, m)); + KASSERT((new.flags & PGA_ENQUEUED) == 0 || new.queue != PQ_NONE, + ("%s: setting PGA_ENQUEUED with PQ_NONE in page %p", __func__, m)); + KASSERT(new._bits != old->_bits, + ("%s: bits are unchanged", __func__)); + return (atomic_fcmpset_32(&m->a._bits, &old->_bits, new._bits) != 0); +} + /* * Clear the given bits in the specified page. */ @@ -805,7 +806,7 @@ vm_page_aflag_clear(vm_page_t m, uint16_t bits) * atomic update. Parallel non-atomic updates to the other fields * within this word are handled properly by the atomic update. */ - addr = (void *)&m->a.flags; + addr = (void *)&m->a; val = bits << VM_PAGE_AFLAG_SHIFT; atomic_clear_32(addr, val); } @@ -825,7 +826,7 @@ vm_page_aflag_set(vm_page_t m, uint16_t bits) * atomic update. Parallel non-atomic updates to the other fields * within this word are handled properly by the atomic update. */ - addr = (void *)&m->a.flags; + addr = (void *)&m->a; val = bits << VM_PAGE_AFLAG_SHIFT; atomic_set_32(addr, val); } @@ -841,24 +842,16 @@ static inline bool vm_page_pqstate_cmpset(vm_page_t m, uint32_t oldq, uint32_t newq, uint32_t fflags, uint32_t nflags) { - uint32_t *addr, nval, oval, qsmask; + vm_page_astate_t new, old; - fflags <<= VM_PAGE_AFLAG_SHIFT; - nflags <<= VM_PAGE_AFLAG_SHIFT; - newq <<= VM_PAGE_QUEUE_SHIFT; - oldq <<= VM_PAGE_QUEUE_SHIFT; - qsmask = ((PGA_DEQUEUE | PGA_REQUEUE | PGA_REQUEUE_HEAD) << - VM_PAGE_AFLAG_SHIFT) | VM_PAGE_QUEUE_MASK; - - addr = (void *)&m->a.flags; - oval = atomic_load_32(addr); + old = vm_page_astate_load(m); do { - if ((oval & fflags) != 0) + if ((old.flags & fflags) != 0 || old.queue != oldq) return (false); - if ((oval & VM_PAGE_QUEUE_MASK) != oldq) - return (false); - nval = (oval & ~qsmask) | nflags | newq; - } while (!atomic_fcmpset_32(addr, &oval, nval)); + new = old; + new.flags = (new.flags & ~PGA_QUEUE_OP_MASK) | nflags; + new.queue = newq; + } while (!vm_page_astate_fcmpset(m, &old, new)); return (true); } _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
