Author: mjg
Date: Sun Feb 5 13:37:23 2017
New Revision: 313282
URL: https://svnweb.freebsd.org/changeset/base/313282
Log:
rwlock: move lockstat handling out of inline primitives
See r313275 for details.
One difference here is that recursion handling was removed from the fallback
routine. As it is it was never supposed to see a recursed lock in the first
place. Future changes will move it out of inline variants, but right now
there is no easy to way to test if the lock is recursed without reading
additional words.
Modified:
head/sys/kern/kern_rwlock.c
head/sys/sys/rwlock.h
Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Sun Feb 5 13:24:54 2017 (r313281)
+++ head/sys/kern/kern_rwlock.c Sun Feb 5 13:37:23 2017 (r313282)
@@ -283,6 +283,7 @@ void
_rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
{
struct rwlock *rw;
+ uintptr_t tid, v;
if (SCHEDULER_STOPPED())
return;
@@ -296,7 +297,14 @@ _rw_wlock_cookie(volatile uintptr_t *c,
("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
line, NULL);
- __rw_wlock(rw, curthread, file, line);
+ tid = (uintptr_t)curthread;
+ v = RW_UNLOCKED;
+ if (!_rw_write_lock_fetch(rw, &v, tid))
+ _rw_wlock_hard(rw, v, tid, file, line);
+ else
+ LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
+ 0, 0, file, line, LOCKSTAT_WRITER);
+
LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_INC(curthread);
@@ -355,7 +363,11 @@ _rw_wunlock_cookie(volatile uintptr_t *c
WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
line);
- __rw_wunlock(rw, curthread, file, line);
+ if (rw->rw_recurse)
+ rw->rw_recurse--;
+ else
+ _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
+
TD_LOCKS_DEC(curthread);
}
@@ -998,13 +1010,12 @@ __rw_wunlock_hard(volatile uintptr_t *c,
return;
rw = rwlock2rw(c);
+ MPASS(!rw_recursed(rw));
- if (rw_wlocked(rw) && rw_recursed(rw)) {
- rw->rw_recurse--;
- if (LOCK_LOG_TEST(&rw->lock_object, 0))
- CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
+ LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,
+ LOCKSTAT_WRITER);
+ if (_rw_write_unlock(rw, tid))
return;
- }
KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
("%s: neither of the waiter flags are set", __func__));
Modified: head/sys/sys/rwlock.h
==============================================================================
--- head/sys/sys/rwlock.h Sun Feb 5 13:24:54 2017 (r313281)
+++ head/sys/sys/rwlock.h Sun Feb 5 13:37:23 2017 (r313282)
@@ -84,10 +84,8 @@
#define _rw_write_lock(rw, tid)
\
atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
-#define _rw_write_lock_fetch(rw, vp, tid) ({
\
- *vp = RW_UNLOCKED; \
- atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)); \
-})
+#define _rw_write_lock_fetch(rw, vp, tid)
\
+ atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid))
/* Release a write lock quickly if there are no waiters. */
#define _rw_write_unlock(rw, tid)
\
@@ -102,13 +100,11 @@
/* Acquire a write lock. */
#define __rw_wlock(rw, tid, file, line) do {
\
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = RW_UNLOCKED; \
\
- if (!_rw_write_lock_fetch((rw), &_v, _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \
+ !_rw_write_lock_fetch((rw), &_v, _tid))) \
_rw_wlock_hard((rw), _v, _tid, (file), (line)); \
- else \
- LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \
- 0, 0, file, line, LOCKSTAT_WRITER); \
} while (0)
/* Release a write lock. */
@@ -118,9 +114,8 @@
if ((rw)->rw_recurse) \
(rw)->rw_recurse--; \
else { \
- LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \
- LOCKSTAT_WRITER); \
- if (!_rw_write_unlock((rw), _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\
+ !_rw_write_unlock((rw), _tid))) \
_rw_wunlock_hard((rw), _tid, (file), (line)); \
} \
} while (0)
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
