Author: jeff Date: Tue Oct 29 20:58:46 2019 New Revision: 354157 URL: https://svnweb.freebsd.org/changeset/base/354157
Log: Use atomics and a shared object lock to protect the object reference count. Certain consumers still need to guarantee a stable reference so we can not switch entirely to atomics yet. Exclusive lock holders can still modify and examine the refcount without using the ref api. Reviewed by: kib Tested by: pho Sponsored by: Netflix, Intel Differential Revision: https://reviews.freebsd.org/D21598 Modified: head/sys/sys/refcount.h head/sys/vm/vm_object.c head/sys/vm/vm_object.h head/sys/vm/vnode_pager.c Modified: head/sys/sys/refcount.h ============================================================================== --- head/sys/sys/refcount.h Tue Oct 29 20:46:25 2019 (r354156) +++ head/sys/sys/refcount.h Tue Oct 29 20:58:46 2019 (r354157) @@ -175,4 +175,22 @@ refcount_release_if_not_last(volatile u_int *count) } } +static __inline __result_use_check bool +refcount_release_if_gt(volatile u_int *count, u_int n) +{ + u_int old; + + KASSERT(n > 0, + ("refcount_release_if_gt: Use refcount_release for final ref")); + old = *count; + for (;;) { + if (REFCOUNT_COUNT(old) <= n) + return (false); + if (__predict_false(REFCOUNT_SATURATED(old))) + return (true); + if (atomic_fcmpset_int(count, &old, old - 1)) + return (true); + } +} + #endif /* ! __SYS_REFCOUNT_H__ */ Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Tue Oct 29 20:46:25 2019 (r354156) +++ head/sys/vm/vm_object.c Tue Oct 29 20:58:46 2019 (r354157) @@ -224,8 +224,8 @@ vm_object_zinit(void *mem, int size, int flags) /* These are true for any object that has been freed */ object->type = OBJT_DEAD; - object->ref_count = 0; vm_radix_init(&object->rtree); + refcount_init(&object->ref_count, 0); refcount_init(&object->paging_in_progress, 0); refcount_init(&object->busy, 0); object->resident_page_count = 0; @@ -282,7 +282,7 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, object->size = size; object->domain.dr_policy = NULL; object->generation = 1; - object->ref_count = 1; + refcount_init(&object->ref_count, 1); object->memattr = VM_MEMATTR_DEFAULT; object->cred = NULL; object->charge = 0; @@ -444,9 +444,9 @@ vm_object_reference(vm_object_t object) { if (object == NULL) return; - VM_OBJECT_WLOCK(object); + VM_OBJECT_RLOCK(object); vm_object_reference_locked(object); - VM_OBJECT_WUNLOCK(object); + VM_OBJECT_RUNLOCK(object); } /* @@ -461,8 +461,8 @@ vm_object_reference_locked(vm_object_t object) { struct vnode *vp; - VM_OBJECT_ASSERT_WLOCKED(object); - object->ref_count++; + VM_OBJECT_ASSERT_LOCKED(object); + refcount_acquire(&object->ref_count); if (object->type == OBJT_VNODE) { vp = object->handle; vref(vp); @@ -477,24 +477,16 @@ vm_object_vndeallocate(vm_object_t object) { struct vnode *vp = (struct vnode *) object->handle; - VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_VNODE, ("vm_object_vndeallocate: not a vnode object")); KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); -#ifdef INVARIANTS - if (object->ref_count == 0) { - vn_printf(vp, "vm_object_vndeallocate "); - panic("vm_object_vndeallocate: bad object reference count"); - } -#endif - if (!umtx_shm_vnobj_persistent && object->ref_count == 1) + if (refcount_release(&object->ref_count) && + !umtx_shm_vnobj_persistent) umtx_shm_object_terminated(object); - object->ref_count--; - + VM_OBJECT_RUNLOCK(object); /* vrele may need the vnode lock. */ - VM_OBJECT_WUNLOCK(object); vrele(vp); } @@ -513,24 +505,32 @@ void vm_object_deallocate(vm_object_t object) { vm_object_t temp; + bool released; while (object != NULL) { - VM_OBJECT_WLOCK(object); + VM_OBJECT_RLOCK(object); if (object->type == OBJT_VNODE) { vm_object_vndeallocate(object); return; } - KASSERT(object->ref_count != 0, - ("vm_object_deallocate: object deallocated too many times: %d", object->type)); - /* * If the reference count goes to 0 we start calling - * vm_object_terminate() on the object chain. - * A ref count of 1 may be a special case depending on the - * shadow count being 0 or 1. + * vm_object_terminate() on the object chain. A ref count + * of 1 may be a special case depending on the shadow count + * being 0 or 1. These cases require a write lock on the + * object. */ - object->ref_count--; + released = refcount_release_if_gt(&object->ref_count, 2); + VM_OBJECT_RUNLOCK(object); + if (released) + return; + + VM_OBJECT_WLOCK(object); + KASSERT(object->ref_count != 0, + ("vm_object_deallocate: object deallocated too many times: %d", object->type)); + + refcount_release(&object->ref_count); if (object->ref_count > 1) { VM_OBJECT_WUNLOCK(object); return; @@ -558,7 +558,7 @@ vm_object_deallocate(vm_object_t object) /* * Avoid a potential deadlock. */ - object->ref_count++; + refcount_acquire(&object->ref_count); VM_OBJECT_WUNLOCK(object); /* * More likely than not the thread @@ -580,7 +580,7 @@ vm_object_deallocate(vm_object_t object) (robject->type == OBJT_DEFAULT || robject->type == OBJT_SWAP)) { - robject->ref_count++; + refcount_acquire(&robject->ref_count); retry: if (REFCOUNT_COUNT(robject->paging_in_progress) > 0) { VM_OBJECT_WUNLOCK(object); @@ -1223,15 +1223,15 @@ vm_object_shadow( * Don't create the new object if the old object isn't shared. */ if (source != NULL) { - VM_OBJECT_WLOCK(source); + VM_OBJECT_RLOCK(source); if (source->ref_count == 1 && source->handle == NULL && (source->type == OBJT_DEFAULT || source->type == OBJT_SWAP)) { - VM_OBJECT_WUNLOCK(source); + VM_OBJECT_RUNLOCK(source); return; } - VM_OBJECT_WUNLOCK(source); + VM_OBJECT_RUNLOCK(source); } /* @@ -1822,7 +1822,7 @@ vm_object_collapse(vm_object_t object) * Drop the reference count on backing_object. Since * its ref_count was at least 2, it will not vanish. */ - backing_object->ref_count--; + refcount_release(&backing_object->ref_count); VM_OBJECT_WUNLOCK(backing_object); counter_u64_add(object_bypasses, 1); } Modified: head/sys/vm/vm_object.h ============================================================================== --- head/sys/vm/vm_object.h Tue Oct 29 20:46:25 2019 (r354156) +++ head/sys/vm/vm_object.h Tue Oct 29 20:58:46 2019 (r354157) @@ -106,7 +106,7 @@ struct vm_object { vm_pindex_t size; /* Object size */ struct domainset_ref domain; /* NUMA policy. */ int generation; /* generation ID */ - int ref_count; /* How many refs?? */ + volatile u_int ref_count; /* How many refs?? */ int shadow_count; /* how many objects that this is a shadow for */ vm_memattr_t memattr; /* default memory attribute for pages */ objtype_t type; /* type of pager */ Modified: head/sys/vm/vnode_pager.c ============================================================================== --- head/sys/vm/vnode_pager.c Tue Oct 29 20:46:25 2019 (r354156) +++ head/sys/vm/vnode_pager.c Tue Oct 29 20:58:46 2019 (r354157) @@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$"); #include <sys/ktr.h> #include <sys/limits.h> #include <sys/conf.h> +#include <sys/refcount.h> #include <sys/rwlock.h> #include <sys/sf_buf.h> #include <sys/domainset.h> @@ -172,9 +173,9 @@ vnode_create_vobject(struct vnode *vp, off_t isize, st * Dereference the reference we just created. This assumes * that the object is associated with the vp. */ - VM_OBJECT_WLOCK(object); - object->ref_count--; - VM_OBJECT_WUNLOCK(object); + VM_OBJECT_RLOCK(object); + refcount_release(&object->ref_count); + VM_OBJECT_RUNLOCK(object); vrele(vp); KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object")); @@ -285,7 +286,7 @@ retry: KASSERT(object->ref_count == 1, ("leaked ref %p %d", object, object->ref_count)); object->type = OBJT_DEAD; - object->ref_count = 0; + refcount_init(&object->ref_count, 0); VM_OBJECT_WUNLOCK(object); vm_object_destroy(object); goto retry; @@ -294,7 +295,7 @@ retry: VI_UNLOCK(vp); } else { VM_OBJECT_WLOCK(object); - object->ref_count++; + refcount_acquire(&object->ref_count); #if VM_NRESERVLEVEL > 0 vm_object_color(object, 0); #endif _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
