OK, here it is. Against 2.6.13-rc7-rt1.
Please, test it well. Make sure to turn off any debugging, especially
CONFIG_RT_DEADLOCK_DETECT. Since that would use a global trace_lock
that defeats the purpose of removing the pi_lock.
The patch to remove the single pi_lock:
-- Steve
Signed-off-by: Steven Rostedt <[EMAIL PROTECTED]>
Index: linux_realtime_goliath/kernel/rt.c
===================================================================
--- linux_realtime_goliath/kernel/rt.c (revision 304)
+++ linux_realtime_goliath/kernel/rt.c (working copy)
@@ -22,6 +22,9 @@
*
* Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt
*
+ * (also by Steven Rostedt)
+ * - Converted single pi_lock to individual task locks.
+ *
*/
#include <linux/config.h>
#include <linux/sched.h>
@@ -36,6 +39,26 @@
#define CAPTURE_LOCK
/*
+ * Lock order:
+ *
+ * To keep from having a single lock for PI, each task and lock
+ * has their own locking. The order is as follows:
+ *
+ * blocked task->pi_lock -> lock->wait_lock -> owner task->pi_lock.
+ *
+ * This is safe since a owner task should never block on a lock that
+ * is owned by a blocking task. Otherwise you would have a deadlock
+ * in the normal system.
+ * The same goes for the locks. A lock held by one task, should not be
+ * taken by task that holds a lock that is blocking this lock's owner.
+ *
+ * A task that is about to grab a lock is first considered to be a
+ * blocking task, even if the task successfully acquires the lock.
+ * This is because the taking of the locks happen before the
+ * task becomes the owner.
+ */
+
+/*
* These flags are used for allowing of stealing of ownerships.
*/
#define RT_PENDOWNER 1 /* pending owner on a lock */
@@ -83,14 +106,6 @@
*/
//#define ALL_TASKS_PI
-/*
- * We need a global lock for priority inheritance handling.
- * This is only for the slow path, but still, we might want
- * to optimize it later to be more scalable.
- */
-static __cacheline_aligned_in_smp raw_spinlock_t pi_lock =
- RAW_SPIN_LOCK_UNLOCKED;
-
#ifdef CONFIG_RT_DEADLOCK_DETECT
# define __EIP_DECL__ , unsigned long eip
# define __EIP__ , eip
@@ -174,6 +189,8 @@
if (trace_on) { \
trace_on = 0; \
console_verbose(); \
+ if (spin_is_locked(¤t->pi_lock)) \
+ __raw_spin_unlock(¤t->pi_lock); \
spin_unlock(&trace_lock); \
} \
} while (0)
@@ -228,7 +245,6 @@
*/
void zap_rt_locks(void)
{
- spin_lock_init(&pi_lock);
#ifdef CONFIG_RT_DEADLOCK_DETECT
spin_lock_init(&trace_lock);
#endif
@@ -589,11 +605,11 @@
printk("exiting task is not even the owner??\n");
goto restart;
}
- __raw_spin_lock(&pi_lock);
+ __raw_spin_lock(&task->pi_lock);
plist_for_each(curr1, &task->pi_waiters) {
w = plist_entry(curr1, struct rt_mutex_waiter, pi_list);
TRACE_OFF();
- __raw_spin_unlock(&pi_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
printk("hm, PI interest held at exit time? Task:\n");
@@ -601,7 +617,7 @@
printk_waiter(w);
return;
}
- __raw_spin_unlock(&pi_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
}
@@ -674,6 +690,7 @@
struct rt_mutex_waiter *w;
struct plist *curr1;
+ __raw_spin_lock(&old_owner->task->pi_lock);
plist_for_each(curr1, &old_owner->task->pi_waiters) {
w = plist_entry(curr1, struct rt_mutex_waiter, pi_list);
if (w->lock == lock) {
@@ -682,9 +699,10 @@
printk_waiter(w);
printk("\n");
TRACE_WARN_ON(1);
- return;
+ break;
}
}
+ __raw_spin_unlock(&old_owner->task->pi_lock);
}
#else
@@ -732,13 +750,24 @@
int pi_walk, pi_null, pi_prio, pi_initialized;
-static void pi_setprio(struct rt_mutex *lock, struct task_struct *p, int prio)
+/*
+ * The lock->wait_lock and p->pi_lock must be held.
+ */
+static void pi_setprio(struct rt_mutex *lock, struct task_struct *task, int
prio)
{
+ struct rt_mutex *l = lock;
+ struct task_struct *p = task;
+ /*
+ * We don't want to release the parameters locks.
+ */
+
if (unlikely(!p->pid)) {
pi_null++;
return;
}
+ TRACE_BUG_ON_LOCKED(!spin_is_locked(&lock->wait_lock));
+ TRACE_BUG_ON_LOCKED(!spin_is_locked(&p->pi_lock));
#ifdef CONFIG_RT_DEADLOCK_DETECT
pi_prio++;
if (p->policy != SCHED_NORMAL && prio > normal_prio(p)) {
@@ -759,7 +788,6 @@
* If the task is blocked on some other task then boost that
* other task (or tasks) too:
*/
- __raw_spin_lock(&p->pi_lock);
for (;;) {
struct rt_mutex_waiter *w = p->blocked_on;
#ifdef CONFIG_RT_DEADLOCK_DETECT
@@ -774,16 +802,27 @@
* it RT, then register the task in the PI list and
* requeue it to the wait list:
*/
- lock = w->lock;
+
+ /*
+ * Don't unlock the original lock->wait_lock
+ */
+ if (l != lock)
+ __raw_spin_unlock(&l->wait_lock);
+ l = w->lock;
TRACE_BUG_ON_LOCKED(!lock);
- TRACE_BUG_ON_LOCKED(!lock_owner(lock));
+
+ if (l != lock)
+ __raw_spin_lock(&l->wait_lock);
+
+ TRACE_BUG_ON_LOCKED(!lock_owner(l));
if (rt_task(p) && plist_empty(&w->pi_list)) {
+ TRACE_BUG_ON_LOCKED(was_rt);
plist_init(&w->pi_list, prio);
- plist_add(&w->pi_list,
&lock_owner(lock)->task->pi_waiters);
+ plist_add(&w->pi_list,
&lock_owner(l)->task->pi_waiters);
- plist_del(&w->list, &lock->wait_list);
+ plist_del(&w->list, &l->wait_list);
plist_init(&w->list, prio);
- plist_add(&w->list, &lock->wait_list);
+ plist_add(&w->list, &l->wait_list);
}
/*
* If the task is blocked on a lock, and we just restored
@@ -795,16 +834,18 @@
*/
if (!rt_task(p) && !plist_empty(&w->pi_list)) {
TRACE_BUG_ON_LOCKED(!was_rt);
- plist_del(&w->pi_list,
&lock_owner(lock)->task->pi_waiters);
- plist_del(&w->list, &lock->wait_list);
+ plist_del(&w->pi_list,
&lock_owner(l)->task->pi_waiters);
+ plist_del(&w->list, &l->wait_list);
plist_init(&w->list, prio);
- plist_add(&w->list, &lock->wait_list);
+ plist_add(&w->list, &l->wait_list);
}
pi_walk++;
+
+ if (p != task)
+ __raw_spin_unlock(&p->pi_lock);
- __raw_spin_unlock(&p->pi_lock);
- p = lock_owner(lock)->task;
+ p = lock_owner(l)->task;
TRACE_BUG_ON_LOCKED(!p);
__raw_spin_lock(&p->pi_lock);
/*
@@ -815,7 +856,10 @@
if (p->prio <= prio)
break;
}
- __raw_spin_unlock(&p->pi_lock);
+ if (l != lock)
+ __raw_spin_unlock(&l->wait_lock);
+ if (p != task)
+ __raw_spin_unlock(&p->pi_lock);
}
/*
@@ -831,7 +875,9 @@
unsigned long flags;
int oldprio;
- spin_lock_irqsave(&pi_lock, flags);
+ spin_lock_irqsave(&p->pi_lock,flags);
+ if (p->blocked_on)
+ spin_lock(&p->blocked_on->lock->wait_lock);
oldprio = p->normal_prio;
if (oldprio == prio)
@@ -858,10 +904,17 @@
else
mutex_setprio(p, prio);
out:
- spin_unlock_irqrestore(&pi_lock, flags);
+ if (p->blocked_on)
+ spin_unlock(&p->blocked_on->lock->wait_lock);
+ spin_unlock_irqrestore(&p->pi_lock, flags);
+
}
+/*
+ * This is called with both the waiter->task->pi_lock and
+ * lock->wait_lock held.
+ */
static void
task_blocks_on_lock(struct rt_mutex_waiter *waiter, struct thread_info *ti,
struct rt_mutex *lock __EIP_DECL__)
@@ -872,7 +925,6 @@
/* mark the current thread as blocked on the lock */
waiter->eip = eip;
#endif
- __raw_spin_lock(&task->pi_lock);
task->blocked_on = waiter;
waiter->lock = lock;
waiter->ti = ti;
@@ -884,31 +936,24 @@
if (!rt_task(task)) {
plist_add(&waiter->list, &lock->wait_list);
set_lock_owner_pending(lock);
- __raw_spin_unlock(&task->pi_lock);
return;
}
#endif
- __raw_spin_unlock(&task->pi_lock);
- __raw_spin_lock(&pi_lock);
+ plist_add(&waiter->pi_list, &lock_owner(lock)->task->pi_waiters);
/*
- * We could have been added to the pi list from another task
- * doing a pi_setprio.
+ * Add RT tasks to the head:
*/
- if (likely(plist_empty(&waiter->pi_list))) {
- plist_add(&waiter->pi_list,
&lock_owner(lock)->task->pi_waiters);
- /*
- * Add RT tasks to the head:
- */
- plist_add(&waiter->list, &lock->wait_list);
- }
+ plist_add(&waiter->list, &lock->wait_list);
set_lock_owner_pending(lock);
/*
* If the waiter has higher priority than the owner
* then temporarily boost the owner:
*/
- if (task->prio < lock_owner(lock)->task->prio)
+ if (task->prio < lock_owner(lock)->task->prio) {
+ __raw_spin_lock(&lock_owner(lock)->task->pi_lock);
pi_setprio(lock, lock_owner(lock)->task, task->prio);
- __raw_spin_unlock(&pi_lock);
+ __raw_spin_unlock(&lock_owner(lock)->task->pi_lock);
+ }
}
/*
@@ -942,6 +987,10 @@
}
EXPORT_SYMBOL(__init_rwsem);
+/*
+ * This must be called with both the old_owner and new_owner pi_locks held.
+ * As well as the lock->wait_lock.
+ */
static inline
void set_new_owner(struct rt_mutex *lock, struct thread_info *old_owner,
struct thread_info *new_owner __EIP_DECL__)
@@ -975,6 +1024,8 @@
/*
* handle the lock release when processes blocked on it that can now run
* - the spinlock must be held by the caller
+ *
+ * The lock->wait_lock must be held, and the lock's owner->pi_lock must not.
*/
static struct thread_info *
pick_new_owner(struct rt_mutex *lock, struct thread_info *old_owner,
@@ -990,14 +1041,34 @@
*/
waiter = plist_first_entry(&lock->wait_list, struct rt_mutex_waiter,
list);
+ try_again:
trace_special_pid(waiter->ti->task->pid, waiter->ti->task->prio, 0);
#ifdef ALL_TASKS_PI
check_pi_list_present(lock, waiter, old_owner);
#endif
new_owner = waiter->ti;
+ /*
+ * The new owner is still blocked on this lock, so we
+ * must release the lock->wait_lock before grabing
+ * the new_owner lock.
+ */
+ __raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_lock(&new_owner->task->pi_lock);
+ __raw_spin_lock(&lock->wait_lock);
+ /*
+ * In this split second of releasing the lock, a high priority
+ * process could have come along and blocked as well.
+ */
+ waiter = plist_first_entry(&lock->wait_list, struct rt_mutex_waiter,
list);
+ if (unlikely(waiter->ti != new_owner)) {
+ __raw_spin_unlock(&new_owner->task->pi_lock);
+ goto try_again;
+ }
plist_del_init(&waiter->list, &lock->wait_list);
+ __raw_spin_lock(&old_owner->task->pi_lock);
+
plist_del(&waiter->pi_list, &old_owner->task->pi_waiters);
plist_init(&waiter->pi_list, waiter->ti->task->prio);
@@ -1008,6 +1079,9 @@
new_owner->task->blocked_on = NULL;
TRACE_WARN_ON(save_state != lock->save_state);
+ __raw_spin_unlock(&old_owner->task->pi_lock);
+ __raw_spin_unlock(&new_owner->task->pi_lock);
+
return new_owner;
}
@@ -1060,9 +1134,7 @@
* is still sleeping and hasn't woken up to get the lock.
*/
- /* Test the simple case first, is it already running? */
- if (!TASK_PENDING(owner))
- return 0;
+ TRACE_BUG_ON_LOCKED(!owner);
/* The owner is pending on a lock, but is it this lock? */
if (owner->pending_owner != lock)
@@ -1134,17 +1206,35 @@
return 0;
trace_lock_irqsave(&trace_lock, flags, ti);
+ /*
+ * We are no longer blocked on the lock, so we are considered a
+ * owner. So we must grab the lock->wait_lock first.
+ */
__raw_spin_lock(&lock->wait_lock);
+ __raw_spin_lock(&task->pi_lock);
if (!(task->rt_flags & RT_PENDOWNER)) {
- /* someone else stole it */
+ /*
+ * Someone else stole it.
+ * We are grabbing a lock now, so we must release the
+ * locks again and retake them in the opposite order.
+ */
+ __raw_spin_unlock(&task->pi_lock);
+ __raw_spin_unlock(&lock->wait_lock);
+
+ __raw_spin_lock(&task->pi_lock);
+ __raw_spin_lock(&lock->wait_lock);
+
old_owner = lock_owner(lock);
TRACE_BUG_ON_LOCKED(old_owner == ti);
if (likely(!old_owner) || __grab_lock(lock, task,
old_owner->task)) {
/* we got it back! */
- __raw_spin_lock(&pi_lock);
- set_new_owner(lock, old_owner, ti __W_EIP__(waiter));
- __raw_spin_unlock(&pi_lock);
+ if (old_owner) {
+ __raw_spin_lock(&old_owner->task->pi_lock);
+ set_new_owner(lock, old_owner, ti
__W_EIP__(waiter));
+ __raw_spin_unlock(&old_owner->task->pi_lock);
+ } else
+ set_new_owner(lock, old_owner, ti
__W_EIP__(waiter));
ret = 0;
} else {
/* Add ourselves back to the list */
@@ -1159,6 +1249,7 @@
}
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
return ret;
@@ -1196,6 +1287,7 @@
trace_lock_irqsave(&trace_lock, flags, ti);
TRACE_BUG_ON_LOCKED(!raw_irqs_disabled());
+ __raw_spin_lock(&task->pi_lock);
__raw_spin_lock(&lock->wait_lock);
INIT_WAITER(&waiter);
@@ -1205,10 +1297,14 @@
if (likely(!old_owner) || __grab_lock(lock, task, old_owner->task)) {
/* granted */
TRACE_WARN_ON_LOCKED(!plist_empty(&lock->wait_list) &&
!old_owner);
- __raw_spin_lock(&pi_lock);
- set_new_owner(lock, old_owner, ti __EIP__);
- __raw_spin_unlock(&pi_lock);
+ if (old_owner) {
+ __raw_spin_lock(&old_owner->task->pi_lock);
+ set_new_owner(lock, old_owner, ti __EIP__);
+ __raw_spin_unlock(&old_owner->task->pi_lock);
+ } else
+ set_new_owner(lock, old_owner, ti __EIP__);
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
FREE_WAITER(&waiter);
@@ -1223,6 +1319,7 @@
TRACE_BUG_ON_LOCKED(!raw_irqs_disabled());
/* we don't need to touch the lock struct anymore */
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
might_sleep();
@@ -1273,6 +1370,7 @@
trace_lock_irqsave(&trace_lock, flags, ti);
TRACE_BUG_ON_LOCKED(!raw_irqs_disabled());
+ __raw_spin_lock(&task->pi_lock);
__raw_spin_lock(&lock->wait_lock);
INIT_WAITER(&waiter);
@@ -1282,10 +1380,14 @@
if (likely(!old_owner) || __grab_lock(lock, task, old_owner->task)) {
/* granted */
TRACE_WARN_ON_LOCKED(!plist_empty(&lock->wait_list) &&
!old_owner);
- __raw_spin_lock(&pi_lock);
- set_new_owner(lock, old_owner, ti __EIP__);
- __raw_spin_unlock(&pi_lock);
+ if (old_owner) {
+ __raw_spin_lock(&old_owner->task->pi_lock);
+ set_new_owner(lock, old_owner, ti __EIP__);
+ __raw_spin_unlock(&old_owner->task->pi_lock);
+ } else
+ set_new_owner(lock, old_owner, ti __EIP__);
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
FREE_WAITER(&waiter);
@@ -1306,6 +1408,7 @@
/* we don't need to touch the lock struct anymore */
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock(&trace_lock, ti);
/*
@@ -1396,7 +1499,6 @@
TRACE_WARN_ON_LOCKED(list_empty(&lock->held_list));
list_del_init(&lock->held_list);
#endif
- __raw_spin_lock(&pi_lock);
#ifdef ALL_TASKS_PI
if (plist_empty(&lock->wait_list))
@@ -1409,7 +1511,6 @@
__up_mutex_waiter_nosavestate(lock __EIP__);
} else
lock->owner = NULL;
- __raw_spin_unlock(&pi_lock);
__raw_spin_unlock(&lock->wait_lock);
#ifdef CONFIG_DEBUG_PREEMPT
if (current->lock_count < 0 || current->lock_count >= MAX_LOCK_STACK) {
@@ -1640,6 +1741,7 @@
trace_lock_irqsave(&trace_lock, flags, ti);
TRACE_BUG_ON_LOCKED(!raw_irqs_disabled());
+ __raw_spin_lock(&task->pi_lock);
__raw_spin_lock(&lock->wait_lock);
INIT_WAITER(&waiter);
@@ -1649,10 +1751,14 @@
if (likely(!old_owner) || __grab_lock(lock, task, old_owner->task)) {
/* granted */
TRACE_WARN_ON_LOCKED(!plist_empty(&lock->wait_list) &&
!old_owner);
- __raw_spin_lock(&pi_lock);
- set_new_owner(lock, old_owner, ti __EIP__);
- __raw_spin_unlock(&pi_lock);
+ if (old_owner) {
+ __raw_spin_lock(&old_owner->task->pi_lock);
+ set_new_owner(lock, old_owner, ti __EIP__);
+ __raw_spin_unlock(&old_owner->task->pi_lock);
+ } else
+ set_new_owner(lock, old_owner, ti __EIP__);
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
FREE_WAITER(&waiter);
@@ -1667,6 +1773,7 @@
TRACE_BUG_ON_LOCKED(!raw_irqs_disabled());
/* we don't need to touch the lock struct anymore */
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
might_sleep();
@@ -1684,6 +1791,7 @@
* didnt get the lock - else return success:
*/
trace_lock_irq(&trace_lock, ti);
+ __raw_spin_lock(&task->pi_lock);
__raw_spin_lock(&lock->wait_lock);
if (waiter.ti) {
plist_del_init(&waiter.list, &lock->wait_list);
@@ -1692,17 +1800,16 @@
* (No big problem if our PI effect lingers
* a bit - owner will restore prio.)
*/
- __raw_spin_lock(&pi_lock);
TRACE_WARN_ON_LOCKED(waiter.ti != ti);
TRACE_WARN_ON_LOCKED(current->blocked_on !=
&waiter);
plist_del(&waiter.pi_list, &task->pi_waiters);
plist_init(&waiter.pi_list, task->prio);
waiter.ti = NULL;
current->blocked_on = NULL;
- __raw_spin_unlock(&pi_lock);
ret = -EINTR;
}
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irq(&trace_lock, ti);
break;
}
@@ -1741,6 +1848,7 @@
trace_lock_irqsave(&trace_lock, flags, ti);
TRACE_BUG_ON_LOCKED(!raw_irqs_disabled());
+ __raw_spin_lock(&task->pi_lock);
__raw_spin_lock(&lock->wait_lock);
old_owner = lock_owner(lock);
@@ -1749,13 +1857,18 @@
if (likely(!old_owner) || __grab_lock(lock, task, old_owner->task)) {
/* granted */
TRACE_WARN_ON_LOCKED(!plist_empty(&lock->wait_list) &&
!old_owner);
- __raw_spin_lock(&pi_lock);
- set_new_owner(lock, old_owner, ti __EIP__);
- __raw_spin_unlock(&pi_lock);
+ if (old_owner) {
+ __raw_spin_lock(&old_owner->task->pi_lock);
+ set_new_owner(lock, old_owner, ti __EIP__);
+ __raw_spin_unlock(&old_owner->task->pi_lock);
+ } else
+ set_new_owner(lock, old_owner, ti __EIP__);
+ __raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
ret = 1;
}
-
__raw_spin_unlock(&lock->wait_lock);
+ __raw_spin_unlock(&task->pi_lock);
trace_unlock_irqrestore(&trace_lock, flags, ti);
return ret;
@@ -1817,6 +1930,7 @@
* to the previous priority (or to the next highest prio
* waiter's priority):
*/
+ __raw_spin_lock(&old_owner->pi_lock);
prio = old_owner->normal_prio;
if (unlikely(!plist_empty(&old_owner->pi_waiters))) {
w = plist_first_entry(&old_owner->pi_waiters, struct
rt_mutex_waiter, pi_list);
@@ -1825,6 +1939,7 @@
}
if (unlikely(prio != old_owner->prio))
pi_setprio(lock, old_owner, prio);
+ __raw_spin_unlock(&old_owner->pi_lock);
#ifdef CAPTURE_LOCK
if (lock != &kernel_sem.lock) {
new_owner->rt_flags |= RT_PENDOWNER;
@@ -1851,6 +1966,7 @@
* to the previous priority (or to the next highest prio
* waiter's priority):
*/
+ __raw_spin_lock(&old_owner->pi_lock);
prio = old_owner->normal_prio;
if (unlikely(!plist_empty(&old_owner->pi_waiters))) {
w = plist_first_entry(&old_owner->pi_waiters, struct
rt_mutex_waiter, pi_list);
@@ -1859,6 +1975,7 @@
}
if (unlikely(prio != old_owner->prio))
pi_setprio(lock, old_owner, prio);
+ __raw_spin_unlock(&old_owner->pi_lock);
#ifdef CAPTURE_LOCK
if (lock != &kernel_sem.lock) {
new_owner->rt_flags |= RT_PENDOWNER;
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
