Author: jeff Date: Tue Oct 29 21:06:34 2019 New Revision: 354158 URL: https://svnweb.freebsd.org/changeset/base/354158
Log: Replace OBJ_MIGHTBEDIRTY with a system using atomics. Remove the TMPFS_DIRTY flag and use the same system. This enables further fault locking improvements by allowing more faults to proceed with a shared lock. Reviewed by: kib Tested by: pho Differential Revision: https://reviews.freebsd.org/D22116 Modified: head/sys/fs/nfsclient/nfs_clvnops.c head/sys/fs/nfsserver/nfs_nfsdport.c head/sys/fs/tmpfs/tmpfs_subr.c head/sys/fs/tmpfs/tmpfs_vfsops.c head/sys/fs/tmpfs/tmpfs_vnops.c head/sys/kern/vfs_subr.c head/sys/ufs/ffs/ffs_rawread.c head/sys/vm/vm_fault.c head/sys/vm/vm_object.c head/sys/vm/vm_object.h head/sys/vm/vm_page.c Modified: head/sys/fs/nfsclient/nfs_clvnops.c ============================================================================== --- head/sys/fs/nfsclient/nfs_clvnops.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/fs/nfsclient/nfs_clvnops.c Tue Oct 29 21:06:34 2019 (r354158) @@ -715,7 +715,7 @@ nfs_open(struct vop_open_args *ap) */ if (vp->v_writecount <= -1) { if ((obj = vp->v_object) != NULL && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { + vm_object_mightbedirty(obj)) { VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(obj); Modified: head/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- head/sys/fs/nfsserver/nfs_nfsdport.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/fs/nfsserver/nfs_nfsdport.c Tue Oct 29 21:06:34 2019 (r354158) @@ -1498,8 +1498,7 @@ nfsvno_fsync(struct vnode *vp, u_int64_t off, int cnt, /* * Give up and do the whole thing */ - if (vp->v_object && - (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) { + if (vp->v_object && vm_object_mightbedirty(vp->v_object)) { VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(vp->v_object); @@ -1529,8 +1528,7 @@ nfsvno_fsync(struct vnode *vp, u_int64_t off, int cnt, } lblkno = off / iosize; - if (vp->v_object && - (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) { + if (vp->v_object && vm_object_mightbedirty(vp->v_object)) { VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, off, off + cnt, OBJPC_SYNC); Modified: head/sys/fs/tmpfs/tmpfs_subr.c ============================================================================== --- head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 29 21:06:34 2019 (r354158) @@ -1477,10 +1477,10 @@ tmpfs_check_mtime(struct vnode *vp) KASSERT((obj->flags & (OBJ_TMPFS_NODE | OBJ_TMPFS)) == (OBJ_TMPFS_NODE | OBJ_TMPFS), ("non-tmpfs obj")); /* unlocked read */ - if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) { + if (obj->generation != obj->cleangeneration) { VM_OBJECT_WLOCK(obj); - if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) { - obj->flags &= ~OBJ_TMPFS_DIRTY; + if (obj->generation != obj->cleangeneration) { + obj->cleangeneration = obj->generation; node = VP_TO_TMPFS_NODE(vp); node->tn_status |= TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED; Modified: head/sys/fs/tmpfs/tmpfs_vfsops.c ============================================================================== --- head/sys/fs/tmpfs/tmpfs_vfsops.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/fs/tmpfs/tmpfs_vfsops.c Tue Oct 29 21:06:34 2019 (r354158) @@ -172,7 +172,7 @@ tmpfs_update_mtime(struct mount *mp, bool lazy) * For non-lazy case, we must flush all pending * metadata changes now. */ - if (!lazy || (obj->flags & OBJ_TMPFS_DIRTY) != 0) { + if (!lazy || obj->generation != obj->cleangeneration) { if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, curthread) != 0) continue; Modified: head/sys/fs/tmpfs/tmpfs_vnops.c ============================================================================== --- head/sys/fs/tmpfs/tmpfs_vnops.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/fs/tmpfs/tmpfs_vnops.c Tue Oct 29 21:06:34 2019 (r354158) @@ -1323,7 +1323,7 @@ tmpfs_need_inactive(struct vop_need_inactive_args *ap) goto need; if (vp->v_type == VREG) { obj = vp->v_object; - if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) + if (obj->generation != obj->cleangeneration) goto need; } return (0); Modified: head/sys/kern/vfs_subr.c ============================================================================== --- head/sys/kern/vfs_subr.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/kern/vfs_subr.c Tue Oct 29 21:06:34 2019 (r354158) @@ -3346,7 +3346,7 @@ vinactive(struct vnode *vp, struct thread *td) * pending I/O and dirty pages in the object. */ if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { + vm_object_mightbedirty(obj)) { VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, 0); VM_OBJECT_WUNLOCK(obj); @@ -4406,7 +4406,7 @@ vfs_msync(struct mount *mp, int flags) MNT_VNODE_FOREACH_ACTIVE(vp, mp, mvp) { obj = vp->v_object; - if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 && + if (obj != NULL && vm_object_mightbedirty(obj) && (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) { if (!vget(vp, LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK, @@ -4696,7 +4696,7 @@ vn_need_pageq_flush(struct vnode *vp) MPASS(mtx_owned(VI_MTX(vp))); need = 0; if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) + vm_object_mightbedirty(obj)) need = 1; return (need); } Modified: head/sys/ufs/ffs/ffs_rawread.c ============================================================================== --- head/sys/ufs/ffs/ffs_rawread.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/ufs/ffs/ffs_rawread.c Tue Oct 29 21:06:34 2019 (r354158) @@ -109,7 +109,7 @@ ffs_rawread_sync(struct vnode *vp) if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0 || ((obj = vp->v_object) != NULL && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0)) { + vm_object_mightbedirty(obj))) { VI_UNLOCK(vp); BO_UNLOCK(bo); @@ -140,7 +140,7 @@ ffs_rawread_sync(struct vnode *vp) } /* Attempt to msync mmap() regions to clean dirty mmap */ if ((obj = vp->v_object) != NULL && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { + vm_object_mightbedirty(obj)) { VI_UNLOCK(vp); VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, OBJPC_SYNC); Modified: head/sys/vm/vm_fault.c ============================================================================== --- head/sys/vm/vm_fault.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/vm/vm_fault.c Tue Oct 29 21:06:34 2019 (r354158) @@ -210,7 +210,7 @@ unlock_and_deallocate(struct faultstate *fs) static void vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_prot_t prot, - vm_prot_t fault_type, int fault_flags, bool set_wd) + vm_prot_t fault_type, int fault_flags, bool excl) { bool need_dirty; @@ -226,11 +226,11 @@ vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_p (fault_flags & VM_FAULT_WIRE) == 0) || (fault_flags & VM_FAULT_DIRTY) != 0; - if (set_wd) - vm_object_set_writeable_dirty(m->object); - else + vm_object_set_writeable_dirty(m->object); + + if (!excl) /* - * If two callers of vm_fault_dirty() with set_wd == + * If two callers of vm_fault_dirty() with excl == * FALSE, one for the map entry with MAP_ENTRY_NOSYNC * flag set, other with flag clear, race, it is * possible for the no-NOSYNC thread to see m->dirty @@ -267,7 +267,7 @@ vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_p */ if (need_dirty) vm_page_dirty(m); - if (!set_wd) + if (!excl) vm_page_unlock(m); else if (need_dirty) vm_pager_page_unswapped(m); @@ -758,29 +758,17 @@ RetryFault_oom: /* * Try to avoid lock contention on the top-level object through * special-case handling of some types of page faults, specifically, - * those that are both (1) mapping an existing page from the top- - * level object and (2) not having to mark that object as containing - * dirty pages. Under these conditions, a read lock on the top-level - * object suffices, allowing multiple page faults of a similar type to - * run in parallel on the same top-level object. + * those that are mapping an existing page from the top-level object. + * Under this condition, a read lock on the object suffices, allowing + * multiple page faults of a similar type to run in parallel. */ if (fs.vp == NULL /* avoid locked vnode leak */ && - (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0 && - /* avoid calling vm_object_set_writeable_dirty() */ - ((prot & VM_PROT_WRITE) == 0 || - (fs.first_object->type != OBJT_VNODE && - (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) || - (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0)) { + (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { VM_OBJECT_RLOCK(fs.first_object); - if ((prot & VM_PROT_WRITE) == 0 || - (fs.first_object->type != OBJT_VNODE && - (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) || - (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0) { - rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type, - fault_flags, wired, m_hold); - if (rv == KERN_SUCCESS) - return (rv); - } + rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type, + fault_flags, wired, m_hold); + if (rv == KERN_SUCCESS) + return (rv); if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { VM_OBJECT_RUNLOCK(fs.first_object); VM_OBJECT_WLOCK(fs.first_object); Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/vm/vm_object.c Tue Oct 29 21:06:34 2019 (r354158) @@ -112,10 +112,10 @@ SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_ "Use old (insecure) msync behavior"); static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, - int pagerflags, int flags, boolean_t *clearobjflags, + int pagerflags, int flags, boolean_t *allclean, boolean_t *eio); static boolean_t vm_object_page_remove_write(vm_page_t p, int flags, - boolean_t *clearobjflags); + boolean_t *allclean); static void vm_object_qcollapse(vm_object_t object); static void vm_object_vndeallocate(vm_object_t object); @@ -282,6 +282,7 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, object->size = size; object->domain.dr_policy = NULL; object->generation = 1; + object->cleangeneration = 1; refcount_init(&object->ref_count, 1); object->memattr = VM_MEMATTR_DEFAULT; object->cred = NULL; @@ -769,7 +770,7 @@ vm_object_terminate(vm_object_t object) * page should be flushed, and FALSE otherwise. */ static boolean_t -vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags) +vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean) { vm_page_assert_busied(p); @@ -780,7 +781,7 @@ vm_object_page_remove_write(vm_page_t p, int flags, bo * cleared in this case so we do not have to set them. */ if ((flags & OBJPC_NOSYNC) != 0 && (p->aflags & PGA_NOSYNC) != 0) { - *clearobjflags = FALSE; + *allclean = FALSE; return (FALSE); } else { pmap_remove_write(p); @@ -813,16 +814,11 @@ vm_object_page_clean(vm_object_t object, vm_ooffset_t vm_page_t np, p; vm_pindex_t pi, tend, tstart; int curgeneration, n, pagerflags; - boolean_t clearobjflags, eio, res; + boolean_t eio, res, allclean; VM_OBJECT_ASSERT_WLOCKED(object); - /* - * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE - * objects. The check below prevents the function from - * operating on non-vnode objects. - */ - if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || + if (object->type != OBJT_VNODE || !vm_object_mightbedirty(object) || object->resident_page_count == 0) return (TRUE); @@ -832,7 +828,7 @@ vm_object_page_clean(vm_object_t object, vm_ooffset_t tstart = OFF_TO_IDX(start); tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK); - clearobjflags = tstart == 0 && tend >= object->size; + allclean = tstart == 0 && tend >= object->size; res = TRUE; rescan: @@ -846,32 +842,26 @@ rescan: if (vm_page_none_valid(p)) continue; if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) { - if (object->generation != curgeneration) { - if ((flags & OBJPC_SYNC) != 0) - goto rescan; - else - clearobjflags = FALSE; - } + if (object->generation != curgeneration && + (flags & OBJPC_SYNC) != 0) + goto rescan; np = vm_page_find_least(object, pi); continue; } - if (!vm_object_page_remove_write(p, flags, &clearobjflags)) { + if (!vm_object_page_remove_write(p, flags, &allclean)) { vm_page_xunbusy(p); continue; } n = vm_object_page_collect_flush(object, p, pagerflags, - flags, &clearobjflags, &eio); + flags, &allclean, &eio); if (eio) { res = FALSE; - clearobjflags = FALSE; + allclean = FALSE; } - if (object->generation != curgeneration) { - if ((flags & OBJPC_SYNC) != 0) - goto rescan; - else - clearobjflags = FALSE; - } + if (object->generation != curgeneration && + (flags & OBJPC_SYNC) != 0) + goto rescan; /* * If the VOP_PUTPAGES() did a truncated write, so @@ -887,7 +877,7 @@ rescan: */ if (n == 0) { n = 1; - clearobjflags = FALSE; + allclean = FALSE; } np = vm_page_find_least(object, pi + n); } @@ -895,14 +885,14 @@ rescan: VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0); #endif - if (clearobjflags) - vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY); + if (allclean) + object->cleangeneration = curgeneration; return (res); } static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags, - int flags, boolean_t *clearobjflags, boolean_t *eio) + int flags, boolean_t *allclean, boolean_t *eio) { vm_page_t ma[vm_pageout_page_count], p_first, tp; int count, i, mreq, runlen; @@ -918,7 +908,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_pa tp = vm_page_next(tp); if (tp == NULL || vm_page_tryxbusy(tp) == 0) break; - if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { + if (!vm_object_page_remove_write(tp, flags, allclean)) { vm_page_xunbusy(tp); break; } @@ -928,7 +918,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_pa tp = vm_page_prev(p_first); if (tp == NULL || vm_page_tryxbusy(tp) == 0) break; - if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { + if (!vm_object_page_remove_write(tp, flags, allclean)) { vm_page_xunbusy(tp); break; } @@ -993,7 +983,7 @@ vm_object_sync(vm_object_t object, vm_ooffset_t offset * I/O. */ if (object->type == OBJT_VNODE && - (object->flags & OBJ_MIGHTBEDIRTY) != 0 && + vm_object_mightbedirty(object) != 0 && ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) { VM_OBJECT_WUNLOCK(object); (void) vn_start_write(vp, &mp, V_WAIT); @@ -2130,18 +2120,13 @@ void vm_object_set_writeable_dirty(vm_object_t object) { - VM_OBJECT_ASSERT_WLOCKED(object); - if (object->type != OBJT_VNODE) { - if ((object->flags & OBJ_TMPFS_NODE) != 0) { - KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs")); - vm_object_set_flag(object, OBJ_TMPFS_DIRTY); - } + VM_OBJECT_ASSERT_LOCKED(object); + + /* Only set for vnodes & tmpfs */ + if (object->type != OBJT_VNODE && + (object->flags & OBJ_TMPFS_NODE) == 0) return; - } - object->generation++; - if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) - return; - vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); + atomic_add_int(&object->generation, 1); } /* Modified: head/sys/vm/vm_object.h ============================================================================== --- head/sys/vm/vm_object.h Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/vm/vm_object.h Tue Oct 29 21:06:34 2019 (r354158) @@ -105,7 +105,8 @@ struct vm_object { struct vm_radix rtree; /* root of the resident page radix trie*/ vm_pindex_t size; /* Object size */ struct domainset_ref domain; /* NUMA policy. */ - int generation; /* generation ID */ + volatile int generation; /* generation ID */ + int cleangeneration; /* Generation at clean time */ volatile u_int ref_count; /* How many refs?? */ int shadow_count; /* how many objects that this is a shadow for */ vm_memattr_t memattr; /* default memory attribute for pages */ @@ -188,9 +189,7 @@ struct vm_object { #define OBJ_UMTXDEAD 0x0020 /* umtx pshared was terminated */ #define OBJ_SIZEVNLOCK 0x0040 /* lock vnode to check obj size */ #define OBJ_PG_DTOR 0x0080 /* dont reset object, leave that for dtor */ -#define OBJ_MIGHTBEDIRTY 0x0100 /* object might be dirty, only for vnode */ #define OBJ_TMPFS_NODE 0x0200 /* object belongs to tmpfs VREG node */ -#define OBJ_TMPFS_DIRTY 0x0400 /* dirty tmpfs obj */ #define OBJ_COLORED 0x1000 /* pg_color is defined */ #define OBJ_ONEMAPPING 0x2000 /* One USE (a single, non-forked) mapping flag */ #define OBJ_TMPFS 0x8000 /* has tmpfs vnode allocated */ @@ -307,6 +306,14 @@ vm_object_reserv(vm_object_t object) return (true); } return (false); +} + +static __inline bool +vm_object_mightbedirty(vm_object_t object) +{ + + return (object->type == OBJT_VNODE && + object->generation != object->cleangeneration); } void vm_object_clear_flag(vm_object_t object, u_short bits); Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Tue Oct 29 20:58:46 2019 (r354157) +++ head/sys/vm/vm_page.c Tue Oct 29 21:06:34 2019 (r354158) @@ -1521,7 +1521,7 @@ vm_page_insert_radixdone(vm_page_t m, vm_object_t obje /* * Since we are inserting a new and possibly dirty page, - * update the object's OBJ_MIGHTBEDIRTY flag. + * update the object's generation count. */ if (pmap_page_is_write_mapped(m)) vm_object_set_writeable_dirty(object); @@ -1691,7 +1691,8 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm /* * The object's resident_page_count does not change because we have - * swapped one page for another, but OBJ_MIGHTBEDIRTY. + * swapped one page for another, but the generation count should + * change if the page is dirty. */ if (pmap_page_is_write_mapped(mnew)) vm_object_set_writeable_dirty(object); _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
