The branch main has been updated by kib:
URL:
https://cgit.FreeBSD.org/src/commit/?id=8a5b2db3d81db16e9e6aaea82cc071bdc766e360
commit 8a5b2db3d81db16e9e6aaea82cc071bdc766e360
Author: Konstantin Belousov <k...@freebsd.org>
AuthorDate: 2024-08-09 22:55:36 +0000
Commit: Konstantin Belousov <k...@freebsd.org>
CommitDate: 2024-08-21 15:18:39 +0000
ranglelock_destroy(): do not remove lock entries from under live lock
acquirer
Tested by: markj, pho
Sponsored by: The FreeBSD Foundation
---
sys/kern/kern_rangelock.c | 56 +++++++++++++++++++++++++++++++++++++++--------
1 file changed, 47 insertions(+), 9 deletions(-)
diff --git a/sys/kern/kern_rangelock.c b/sys/kern/kern_rangelock.c
index c2f1f2d762bb..c01ed05e4a6a 100644
--- a/sys/kern/kern_rangelock.c
+++ b/sys/kern/kern_rangelock.c
@@ -289,6 +289,7 @@ static uma_zone_t rl_entry_zone;
static smr_t rl_smr;
static void rangelock_free_free(struct rl_q_entry *free);
+static void rangelock_noncheating_destroy(struct rangelock *lock);
static void
rangelock_sys_init(void)
@@ -340,16 +341,9 @@ rangelock_init(struct rangelock *lock)
void
rangelock_destroy(struct rangelock *lock)
{
- struct rl_q_entry *e, *ep;
-
MPASS(!lock->sleepers);
- if (rangelock_cheat_destroy(lock))
- return;
- for (e = (struct rl_q_entry *)atomic_load_ptr(&lock->head);
- e != NULL; e = rl_e_unmark(ep)) {
- ep = atomic_load_ptr(&e->rl_q_next);
- uma_zfree_smr(rl_entry_zone, e);
- }
+ if (!rangelock_cheat_destroy(lock))
+ rangelock_noncheating_destroy(lock);
}
static bool
@@ -487,6 +481,50 @@ rl_q_cas(struct rl_q_entry **prev, struct rl_q_entry *old,
(uintptr_t)new) != 0);
}
+static void
+rangelock_noncheating_destroy(struct rangelock *lock)
+{
+ struct rl_q_entry *cur, *free, *next, **prev;
+
+ free = NULL;
+again:
+ smr_enter(rl_smr);
+ prev = (struct rl_q_entry **)&lock->head;
+ cur = rl_q_load(prev);
+ MPASS(!rl_e_is_marked(cur));
+
+ for (;;) {
+ if (cur == NULL)
+ break;
+ if (rl_e_is_marked(cur))
+ goto again;
+
+ next = rl_q_load(&cur->rl_q_next);
+ if (rl_e_is_marked(next)) {
+ next = rl_e_unmark(next);
+ if (rl_q_cas(prev, cur, next)) {
+#ifdef INVARIANTS
+ cur->rl_q_owner = NULL;
+#endif
+ cur->rl_q_free = free;
+ free = cur;
+ cur = next;
+ continue;
+ }
+ smr_exit(rl_smr);
+ goto again;
+ }
+
+ sleepq_lock(&lock->sleepers);
+ if (!rl_e_is_marked(cur)) {
+ rl_insert_sleep(lock);
+ goto again;
+ }
+ }
+ smr_exit(rl_smr);
+ rangelock_free_free(free);
+}
+
enum RL_INSERT_RES {
RL_TRYLOCK_FAILED,
RL_LOCK_SUCCESS,
