From: Xunlei Pang <[email protected]> In check_preempt_equal_prio_common(), it requeues "next" ahead in the "run queue" and want to push current away. But when doing the actual pushing, if the system state changes, the pushing may fail as a result.
In this case, p finally becomes the new current and gets running, while previous current was queued back waiting in the same "run queue". This broke FIFO. This patch adds a flag named RT_PREEMPT_PUSHAWAY for task_struct:: rt_preempt, sets it when doing check_preempt_equal_prio_common(), and clears it if current is away(it will call dequeued). So we can test this flag in p's post_schedule_rt() to judge if the pushing has happened. If the pushing failed, requeue previous current back to the head of its "run queue" and start a rescheduling. Signed-off-by: Xunlei Pang <[email protected]> --- kernel/sched/rt.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 7439121..d1cecd6 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -258,6 +258,8 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) #ifdef CONFIG_SMP #define RT_PREEMPT_QUEUEAHEAD 1UL +#define RT_PREEMPT_PUSHAWAY 2UL +#define RT_PREEMPT_MASK 3UL /* * p(current) was preempted, and to be put ahead of @@ -273,6 +275,30 @@ static inline void clear_rt_preempted(struct task_struct *p) p->rt_preempt = 0; } +static inline struct task_struct *rt_preempting_target(struct task_struct *p) +{ + return (struct task_struct *) (p->rt_preempt & ~RT_PREEMPT_MASK); +} + +/* + * p(new current) is preempting and pushing previous current away. + */ +static inline bool rt_preempting(struct task_struct *p) +{ + if ((p->rt_preempt & RT_PREEMPT_PUSHAWAY) && rt_preempting_target(p)) + return true; + + return false; +} + +static inline void clear_rt_preempting(struct task_struct *p) +{ + if (rt_preempting(p)) + put_task_struct(rt_preempting_target(p)); + + p->rt_preempt = 0; +} + void resched_curr_preempted_rt(struct rq *rq) { if (rt_task(rq->curr)) @@ -375,13 +401,17 @@ static inline int has_pushable_tasks(struct rq *rq) return !plist_head_empty(&rq->rt.pushable_tasks); } -static inline void set_post_schedule(struct rq *rq) +static inline void set_post_schedule(struct rq *rq, struct task_struct *p) { - /* - * We detect this state here so that we can avoid taking the RQ - * lock again later if there is no need to push - */ - rq->post_schedule = has_pushable_tasks(rq); + if (rt_preempting(p)) + /* Forced post schedule */ + rq->post_schedule = 1; + else + /* + * We detect this state here so that we can avoid taking + * the RQ lock again later if there is no need to push + */ + rq->post_schedule = has_pushable_tasks(rq); } static void @@ -434,6 +464,15 @@ static inline void clear_rt_preempted(struct task_struct *p) { } +static inline bool rt_preempting(struct task_struct *p) +{ + return false; +} + +static inline void clear_rt_preempting(struct task_struct *p) +{ +} + static inline void resched_curr_preempted_rt(struct rq *rq) { resched_curr(rq); @@ -472,7 +511,7 @@ static inline int pull_rt_task(struct rq *this_rq) return 0; } -static inline void set_post_schedule(struct rq *rq) +static inline void set_post_schedule(struct rq *rq, struct task_struct *p) { } #endif /* CONFIG_SMP */ @@ -1330,6 +1369,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) dequeue_rt_entity(rt_se); dequeue_pushable_task(rq, p); + clear_rt_preempting(p); } /* @@ -1468,6 +1508,11 @@ static void check_preempt_equal_prio_common(struct rq *rq) * to try and push current away. */ requeue_task_rt(rq, next, 1); + + get_task_struct(curr); + curr->rt_preempt |= RT_PREEMPT_PUSHAWAY; + next->rt_preempt = (unsigned long) curr; + next->rt_preempt |= RT_PREEMPT_PUSHAWAY; resched_curr_preempted_rt(rq); } @@ -1590,7 +1635,7 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev) /* The running task is never eligible for pushing */ dequeue_pushable_task(rq, p); - set_post_schedule(rq); + set_post_schedule(rq, p); return p; } @@ -2151,6 +2196,32 @@ skip: static void post_schedule_rt(struct rq *rq) { push_rt_tasks(rq); + + if (rt_preempting(current)) { + struct task_struct *target; + + target = rt_preempting_target(current); + current->rt_preempt = 0; + if (!(target->rt_preempt & RT_PREEMPT_PUSHAWAY)) + goto out; + + /* + * target still has RT_PREEMPT_PUSHAWAY set which + * means it wasn't pushed away successfully if it + * is still on this rq. thus restore former status + * of current and target if so. + */ + if (!task_on_rq_queued(target) || + task_cpu(target) != rq->cpu) + goto out; + + /* target is previous current, requeue it back ahead. */ + requeue_task_rt(rq, target, 1); + /* Let's preempt current, loop back to __schedule(). */ + resched_curr_preempted_rt(rq); +out: + put_task_struct(target); + } } /* -- 1.9.1 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
