Author: mjg
Date: Sun Feb 5 04:53:13 2017
New Revision: 313270
URL: https://svnweb.freebsd.org/changeset/base/313270
Log:
rwlock: switch to fcmpset
Discussed with: jhb
Tested by: pho
Modified:
head/sys/kern/kern_rwlock.c
head/sys/sys/rwlock.h
Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Sun Feb 5 03:26:34 2017 (r313269)
+++ head/sys/kern/kern_rwlock.c Sun Feb 5 04:53:13 2017 (r313270)
@@ -440,7 +440,7 @@ __rw_rlock(volatile uintptr_t *c, const
* if the lock has been unlocked and write waiters
* were present.
*/
- if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
+ if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v,
v + RW_ONE_READER)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR4(KTR_LOCK,
@@ -449,7 +449,6 @@ __rw_rlock(volatile uintptr_t *c, const
(void *)(v + RW_ONE_READER));
break;
}
- v = RW_READ_VALUE(rw);
continue;
}
#ifdef KDTRACE_HOOKS
@@ -675,7 +674,7 @@ _rw_runlock_cookie(volatile uintptr_t *c
* just drop one and return.
*/
if (RW_READERS(x) > 1) {
- if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
+ if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x,
x - RW_ONE_READER)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR4(KTR_LOCK,
@@ -684,7 +683,6 @@ _rw_runlock_cookie(volatile uintptr_t *c
(void *)(x - RW_ONE_READER));
break;
}
- x = RW_READ_VALUE(rw);
continue;
}
/*
@@ -694,14 +692,13 @@ _rw_runlock_cookie(volatile uintptr_t *c
if (!(x & RW_LOCK_WAITERS)) {
MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
RW_READERS_LOCK(1));
- if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
+ if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x,
RW_UNLOCKED)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p last succeeded",
__func__, rw);
break;
}
- x = RW_READ_VALUE(rw);
continue;
}
/*
@@ -769,8 +766,8 @@ _rw_runlock_cookie(volatile uintptr_t *c
* read or write lock.
*/
void
-__rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
- int line)
+__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+ const char *file, int line)
{
struct rwlock *rw;
struct turnstile *ts;
@@ -779,7 +776,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u
int spintries = 0;
int i;
#endif
- uintptr_t v, x;
+ uintptr_t x;
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
@@ -803,7 +800,6 @@ __rw_wlock_hard(volatile uintptr_t *c, u
lock_delay_arg_init(&lda, NULL);
#endif
rw = rwlock2rw(c);
- v = RW_READ_VALUE(rw);
if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) {
KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
@@ -825,9 +821,8 @@ __rw_wlock_hard(volatile uintptr_t *c, u
#endif
for (;;) {
if (v == RW_UNLOCKED) {
- if (_rw_write_lock(rw, tid))
+ if (_rw_write_lock_fetch(rw, &v, tid))
break;
- v = RW_READ_VALUE(rw);
continue;
}
#ifdef KDTRACE_HOOKS
Modified: head/sys/sys/rwlock.h
==============================================================================
--- head/sys/sys/rwlock.h Sun Feb 5 03:26:34 2017 (r313269)
+++ head/sys/sys/rwlock.h Sun Feb 5 04:53:13 2017 (r313270)
@@ -84,6 +84,11 @@
#define _rw_write_lock(rw, tid)
\
atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
+#define _rw_write_lock_fetch(rw, vp, tid) ({
\
+ *vp = RW_UNLOCKED; \
+ atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)); \
+})
+
/* Release a write lock quickly if there are no waiters. */
#define _rw_write_unlock(rw, tid)
\
atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED)
@@ -97,9 +102,10 @@
/* Acquire a write lock. */
#define __rw_wlock(rw, tid, file, line) do {
\
uintptr_t _tid = (uintptr_t)(tid); \
+ uintptr_t _v; \
\
- if ((rw)->rw_lock != RW_UNLOCKED || !_rw_write_lock((rw), _tid))\
- _rw_wlock_hard((rw), _tid, (file), (line)); \
+ if (!_rw_write_lock_fetch((rw), &_v, _tid)) \
+ _rw_wlock_hard((rw), _v, _tid, (file), (line)); \
else \
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \
0, 0, file, line, LOCKSTAT_WRITER); \
@@ -114,7 +120,7 @@
else { \
LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \
LOCKSTAT_WRITER); \
- if ((rw)->rw_lock != _tid || !_rw_write_unlock((rw), _tid))\
+ if (!_rw_write_unlock((rw), _tid)) \
_rw_wunlock_hard((rw), _tid, (file), (line)); \
} \
} while (0)
@@ -135,8 +141,8 @@ void _rw_wunlock_cookie(volatile uintptr
void __rw_rlock(volatile uintptr_t *c, const char *file, int line);
int __rw_try_rlock(volatile uintptr_t *c, const char *file, int line);
void _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line);
-void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
- int line);
+void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+ const char *file, int line);
void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid,
const char *file, int line);
int __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line);
@@ -171,8 +177,8 @@ void __rw_assert(const volatile uintptr_
__rw_try_rlock(&(rw)->rw_lock, f, l)
#define _rw_runlock(rw, f, l)
\
_rw_runlock_cookie(&(rw)->rw_lock, f, l)
-#define _rw_wlock_hard(rw, t, f, l)
\
- __rw_wlock_hard(&(rw)->rw_lock, t, f, l)
+#define _rw_wlock_hard(rw, v, t, f, l)
\
+ __rw_wlock_hard(&(rw)->rw_lock, v, t, f, l)
#define _rw_wunlock_hard(rw, t, f, l)
\
__rw_wunlock_hard(&(rw)->rw_lock, t, f, l)
#define _rw_try_upgrade(rw, f, l)
\
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
