Author: mjg
Date: Sat Feb 18 22:06:03 2017
New Revision: 313928
URL: https://svnweb.freebsd.org/changeset/base/313928
Log:
locks: clean up trylock primitives
In particular thius reduces accesses of the lock itself.
Modified:
head/sys/kern/kern_mutex.c
head/sys/kern/kern_rwlock.c
head/sys/kern/kern_sx.c
Modified: head/sys/kern/kern_mutex.c
==============================================================================
--- head/sys/kern/kern_mutex.c Sat Feb 18 21:59:19 2017 (r313927)
+++ head/sys/kern/kern_mutex.c Sat Feb 18 22:06:03 2017 (r313928)
@@ -374,13 +374,18 @@ int
_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int
line)
{
struct mtx *m;
+ struct thread *td;
+ uintptr_t tid, v;
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
int rval;
+ bool recursed;
- if (SCHEDULER_STOPPED())
+ td = curthread;
+ tid = (uintptr_t)td;
+ if (SCHEDULER_STOPPED_TD(td))
return (1);
m = mtxlock2mtx(c);
@@ -394,13 +399,21 @@ _mtx_trylock_flags_(volatile uintptr_t *
("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
file, line));
- if (mtx_owned(m) && ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
- (opts & MTX_RECURSE) != 0)) {
- m->mtx_recurse++;
- atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
- rval = 1;
- } else
- rval = _mtx_obtain_lock(m, (uintptr_t)curthread);
+ rval = 1;
+ recursed = false;
+ v = MTX_UNOWNED;
+ if (!_mtx_obtain_lock_fetch(m, &v, tid)) {
+ if (v == tid &&
+ ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
+ (opts & MTX_RECURSE) != 0)) {
+ m->mtx_recurse++;
+ atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
+ recursed = true;
+ } else {
+ rval = 0;
+ }
+ }
+
opts &= ~MTX_RECURSE;
LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line);
@@ -408,10 +421,9 @@ _mtx_trylock_flags_(volatile uintptr_t *
WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE |
LOP_TRYLOCK,
file, line);
TD_LOCKS_INC(curthread);
- if (m->mtx_recurse == 0)
+ if (!recursed)
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
m, contested, waittime, file, line);
-
}
return (rval);
Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Sat Feb 18 21:59:19 2017 (r313927)
+++ head/sys/kern/kern_rwlock.c Sat Feb 18 22:06:03 2017 (r313928)
@@ -293,9 +293,14 @@ int
__rw_try_wlock(volatile uintptr_t *c, const char *file, int line)
{
struct rwlock *rw;
+ struct thread *td;
+ uintptr_t tid, v;
int rval;
+ bool recursed;
- if (SCHEDULER_STOPPED())
+ td = curthread;
+ tid = (uintptr_t)td;
+ if (SCHEDULER_STOPPED_TD(td))
return (1);
rw = rwlock2rw(c);
@@ -306,20 +311,23 @@ __rw_try_wlock(volatile uintptr_t *c, co
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
- if (rw_wlocked(rw) &&
- (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
- rw->rw_recurse++;
- atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
- rval = 1;
- } else
- rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
- (uintptr_t)curthread);
+ rval = 1;
+ recursed = false;
+ v = RW_UNLOCKED;
+ if (!atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid)) {
+ if (v == tid && (rw->lock_object.lo_flags & LO_RECURSABLE)) {
+ rw->rw_recurse++;
+ atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
+ } else {
+ rval = 0;
+ }
+ }
LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
if (rval) {
WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
file, line);
- if (!rw_recursed(rw))
+ if (!recursed)
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire,
rw, 0, 0, file, line, LOCKSTAT_WRITER);
TD_LOCKS_INC(curthread);
@@ -637,13 +645,13 @@ __rw_try_rlock(volatile uintptr_t *c, co
("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d",
curthread, rw->lock_object.lo_name, file, line));
+ x = rw->rw_lock;
for (;;) {
- x = rw->rw_lock;
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
if (!(x & RW_LOCK_READ))
break;
- if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
+ if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &x, x +
RW_ONE_READER)) {
LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
line);
WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
Modified: head/sys/kern/kern_sx.c
==============================================================================
--- head/sys/kern/kern_sx.c Sat Feb 18 21:59:19 2017 (r313927)
+++ head/sys/kern/kern_sx.c Sat Feb 18 22:06:03 2017 (r313928)
@@ -269,13 +269,13 @@ sx_try_slock_(struct sx *sx, const char
("sx_try_slock() by idle thread %p on sx %s @ %s:%d",
curthread, sx->lock_object.lo_name, file, line));
+ x = sx->sx_lock;
for (;;) {
- x = sx->sx_lock;
KASSERT(x != SX_LOCK_DESTROYED,
("sx_try_slock() of destroyed sx @ %s:%d", file, line));
if (!(x & SX_LOCK_SHARED))
break;
- if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
+ if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, x +
SX_ONE_SHARER)) {
LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file,
line);
WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
@@ -322,9 +322,14 @@ _sx_xlock(struct sx *sx, int opts, const
int
sx_try_xlock_(struct sx *sx, const char *file, int line)
{
+ struct thread *td;
+ uintptr_t tid, x;
int rval;
+ bool recursed;
- if (SCHEDULER_STOPPED())
+ td = curthread;
+ tid = (uintptr_t)td;
+ if (SCHEDULER_STOPPED_TD(td))
return (1);
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
@@ -333,19 +338,23 @@ sx_try_xlock_(struct sx *sx, const char
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
- if (sx_xlocked(sx) &&
- (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) {
- sx->sx_recurse++;
- atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
- rval = 1;
- } else
- rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
- (uintptr_t)curthread);
+ rval = 1;
+ recursed = false;
+ x = SX_LOCK_UNLOCKED;
+ if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid)) {
+ if (x == tid && (sx->lock_object.lo_flags & LO_RECURSABLE)) {
+ sx->sx_recurse++;
+ atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
+ } else {
+ rval = 0;
+ }
+ }
+
LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
if (rval) {
WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
file, line);
- if (!sx_recursed(sx))
+ if (!recursed)
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
sx, 0, 0, file, line, LOCKSTAT_WRITER);
TD_LOCKS_INC(curthread);
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
