Make the qrwlock code to store an encoded cpu number (+1 saturated)
for the writer that hold the write lock if desired.
Signed-off-by: Waiman Long <long...@redhat.com>
---
include/asm-generic/qrwlock.h | 12 +++++++++++-
kernel/locking/qrwlock.c | 11 ++++++-----
2 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h
index 3aefde23dcea..1b1d5253e314 100644
--- a/include/asm-generic/qrwlock.h
+++ b/include/asm-generic/qrwlock.h
@@ -15,11 +15,21 @@
#include <asm-generic/qrwlock_types.h>
+/*
+ * If __cpu_number_sadd1 (+2 saturated cpu number) is defined, use it as the
+ * writer lock value.
+ */
+#ifdef __cpu_number_sadd1
+#define _QW_LOCKED __cpu_number_sadd1
+#else
+#define _QW_LOCKED 0xff
+#endif
+
/*
* Writer states & reader shift and bias.
*/
#define _QW_WAITING 0x100 /* A writer is waiting */
-#define _QW_LOCKED 0x0ff /* A writer holds the lock */
+#define _QW_LMASK 0x0ff /* A writer lock byte mask */
#define _QW_WMASK 0x1ff /* Writer mask */
#define _QR_SHIFT 9 /* Reader count shift */
#define _QR_BIAS (1U << _QR_SHIFT)
diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index fe9ca92faa2a..394f34db4b8f 100644
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -30,7 +30,7 @@ void queued_read_lock_slowpath(struct qrwlock *lock)
* so spin with ACQUIRE semantics until the lock is available
* without waiting in the queue.
*/
- atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
+ atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LMASK));
return;
}
atomic_sub(_QR_BIAS, &lock->cnts);
@@ -46,7 +46,7 @@ void queued_read_lock_slowpath(struct qrwlock *lock)
* that accesses can't leak upwards out of our subsequent critical
* section in the case that the lock is currently held for write.
*/
- atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
+ atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LMASK));
/*
* Signal the next one in queue to become queue head
@@ -61,12 +61,14 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
*/
void queued_write_lock_slowpath(struct qrwlock *lock)
{
+ const u8 lockval = _QW_LOCKED;
+
/* Put the writer into the wait queue */
arch_spin_lock(&lock->wait_lock);
/* Try to acquire the lock directly if no reader is present */
if (!atomic_read(&lock->cnts) &&
- (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
+ (atomic_cmpxchg_acquire(&lock->cnts, 0, lockval) == 0))
goto unlock;
/* Set the waiting flag to notify readers that a writer is pending */
@@ -75,8 +77,7 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
/* When no more readers or writers, set the locked flag */
do {
atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING);
- } while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING,
- _QW_LOCKED) != _QW_WAITING);
+ } while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING, lockval) !=
_QW_WAITING);
unlock:
arch_spin_unlock(&lock->wait_lock);
}
--
2.18.1
