[PATCH v3 05/17] pull RT tasks
This patch adds the algorithm to pull tasks from RT overloaded runqueues.
When a pull RT is initiated, all overloaded runqueues are examined for
a RT task that is higher in prio than the highest prio task queued on the
target runqueue. If another runqueue holds a RT task that is of higher
prio than the highest prio task on the target runqueue is found it is pulled
to the target runqueue.
Signed-off-by: Steven Rostedt <[EMAIL PROTECTED]>
---
kernel/sched.c|2
kernel/sched_rt.c | 196 ++
2 files changed, 187 insertions(+), 11 deletions(-)
Index: linux-compile.git/kernel/sched.c
===
--- linux-compile.git.orig/kernel/sched.c 2007-11-16 22:11:37.0
-0500
+++ linux-compile.git/kernel/sched.c2007-11-16 22:12:13.0 -0500
@@ -3646,6 +3646,8 @@ need_resched_nonpreemptible:
switch_count = >nvcsw;
}
+ schedule_balance_rt(rq, prev);
+
if (unlikely(!rq->nr_running))
idle_balance(cpu, rq);
Index: linux-compile.git/kernel/sched_rt.c
===
--- linux-compile.git.orig/kernel/sched_rt.c2007-11-16 22:12:07.0
-0500
+++ linux-compile.git/kernel/sched_rt.c 2007-11-16 22:12:13.0 -0500
@@ -176,8 +176,17 @@ static void put_prev_task_rt(struct rq *
static int double_lock_balance(struct rq *this_rq, struct rq *busiest);
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep);
+static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
+{
+ if (!task_running(rq, p) &&
+ (cpu < 0 || cpu_isset(cpu, p->cpus_allowed)))
+ return 1;
+ return 0;
+}
+
/* Return the second highest RT task, NULL otherwise */
-static struct task_struct *pick_next_highest_task_rt(struct rq *rq)
+static struct task_struct *pick_next_highest_task_rt(struct rq *rq,
+int cpu)
{
struct rt_prio_array *array = >rt.active;
struct task_struct *next;
@@ -196,26 +205,36 @@ static struct task_struct *pick_next_hig
}
queue = array->queue + idx;
+ BUG_ON(list_empty(queue));
+
next = list_entry(queue->next, struct task_struct, run_list);
- if (unlikely(next != rq->curr))
- return next;
+ if (unlikely(pick_rt_task(rq, next, cpu)))
+ goto out;
if (queue->next->next != queue) {
/* same prio task */
next = list_entry(queue->next->next, struct task_struct,
run_list);
- return next;
+ if (pick_rt_task(rq, next, cpu))
+ goto out;
}
+ retry:
/* slower, but more flexible */
idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1);
- if (unlikely(idx >= MAX_RT_PRIO)) {
- WARN_ON(1); /* rt_nr_running was 2 and above! */
+ if (unlikely(idx >= MAX_RT_PRIO))
return NULL;
- }
queue = array->queue + idx;
- next = list_entry(queue->next, struct task_struct, run_list);
+ BUG_ON(list_empty(queue));
+ list_for_each_entry(next, queue, run_list) {
+ if (pick_rt_task(rq, next, cpu))
+ goto out;
+ }
+
+ goto retry;
+
+ out:
return next;
}
@@ -300,13 +319,15 @@ static int push_rt_task(struct rq *this_
assert_spin_locked(_rq->lock);
- next_task = pick_next_highest_task_rt(this_rq);
+ next_task = pick_next_highest_task_rt(this_rq, -1);
if (!next_task)
return 0;
retry:
- if (unlikely(next_task == this_rq->curr))
+ if (unlikely(next_task == this_rq->curr)) {
+ WARN_ON(1);
return 0;
+ }
/*
* It's possible that the next_task slipped in of
@@ -330,7 +351,7 @@ static int push_rt_task(struct rq *this_
* so it is possible that next_task has changed.
* If it has, then try again.
*/
- task = pick_next_highest_task_rt(this_rq);
+ task = pick_next_highest_task_rt(this_rq, -1);
if (unlikely(task != next_task) && task && paranoid--) {
put_task_struct(next_task);
next_task = task;
@@ -373,6 +394,158 @@ static void push_rt_tasks(struct rq *rq)
;
}
+static int pull_rt_task(struct rq *this_rq)
+{
+ struct task_struct *next;
+ struct task_struct *p;
+ struct rq *src_rq;
+ cpumask_t *rto_cpumask;
+ int this_cpu = this_rq->cpu;
+ int cpu;
+ int ret = 0;
+
+ assert_spin_locked(_rq->lock);
+
+ /*
+* If cpusets are used, and we have overlapping
+* run queue cpusets, then this algorithm may not catch all.
+* This is just the
[PATCH v3 05/17] pull RT tasks
This patch adds the algorithm to pull tasks from RT overloaded runqueues.
When a pull RT is initiated, all overloaded runqueues are examined for
a RT task that is higher in prio than the highest prio task queued on the
target runqueue. If another runqueue holds a RT task that is of higher
prio than the highest prio task on the target runqueue is found it is pulled
to the target runqueue.
Signed-off-by: Steven Rostedt [EMAIL PROTECTED]
---
kernel/sched.c|2
kernel/sched_rt.c | 196 ++
2 files changed, 187 insertions(+), 11 deletions(-)
Index: linux-compile.git/kernel/sched.c
===
--- linux-compile.git.orig/kernel/sched.c 2007-11-16 22:11:37.0
-0500
+++ linux-compile.git/kernel/sched.c2007-11-16 22:12:13.0 -0500
@@ -3646,6 +3646,8 @@ need_resched_nonpreemptible:
switch_count = prev-nvcsw;
}
+ schedule_balance_rt(rq, prev);
+
if (unlikely(!rq-nr_running))
idle_balance(cpu, rq);
Index: linux-compile.git/kernel/sched_rt.c
===
--- linux-compile.git.orig/kernel/sched_rt.c2007-11-16 22:12:07.0
-0500
+++ linux-compile.git/kernel/sched_rt.c 2007-11-16 22:12:13.0 -0500
@@ -176,8 +176,17 @@ static void put_prev_task_rt(struct rq *
static int double_lock_balance(struct rq *this_rq, struct rq *busiest);
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep);
+static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
+{
+ if (!task_running(rq, p)
+ (cpu 0 || cpu_isset(cpu, p-cpus_allowed)))
+ return 1;
+ return 0;
+}
+
/* Return the second highest RT task, NULL otherwise */
-static struct task_struct *pick_next_highest_task_rt(struct rq *rq)
+static struct task_struct *pick_next_highest_task_rt(struct rq *rq,
+int cpu)
{
struct rt_prio_array *array = rq-rt.active;
struct task_struct *next;
@@ -196,26 +205,36 @@ static struct task_struct *pick_next_hig
}
queue = array-queue + idx;
+ BUG_ON(list_empty(queue));
+
next = list_entry(queue-next, struct task_struct, run_list);
- if (unlikely(next != rq-curr))
- return next;
+ if (unlikely(pick_rt_task(rq, next, cpu)))
+ goto out;
if (queue-next-next != queue) {
/* same prio task */
next = list_entry(queue-next-next, struct task_struct,
run_list);
- return next;
+ if (pick_rt_task(rq, next, cpu))
+ goto out;
}
+ retry:
/* slower, but more flexible */
idx = find_next_bit(array-bitmap, MAX_RT_PRIO, idx+1);
- if (unlikely(idx = MAX_RT_PRIO)) {
- WARN_ON(1); /* rt_nr_running was 2 and above! */
+ if (unlikely(idx = MAX_RT_PRIO))
return NULL;
- }
queue = array-queue + idx;
- next = list_entry(queue-next, struct task_struct, run_list);
+ BUG_ON(list_empty(queue));
+ list_for_each_entry(next, queue, run_list) {
+ if (pick_rt_task(rq, next, cpu))
+ goto out;
+ }
+
+ goto retry;
+
+ out:
return next;
}
@@ -300,13 +319,15 @@ static int push_rt_task(struct rq *this_
assert_spin_locked(this_rq-lock);
- next_task = pick_next_highest_task_rt(this_rq);
+ next_task = pick_next_highest_task_rt(this_rq, -1);
if (!next_task)
return 0;
retry:
- if (unlikely(next_task == this_rq-curr))
+ if (unlikely(next_task == this_rq-curr)) {
+ WARN_ON(1);
return 0;
+ }
/*
* It's possible that the next_task slipped in of
@@ -330,7 +351,7 @@ static int push_rt_task(struct rq *this_
* so it is possible that next_task has changed.
* If it has, then try again.
*/
- task = pick_next_highest_task_rt(this_rq);
+ task = pick_next_highest_task_rt(this_rq, -1);
if (unlikely(task != next_task) task paranoid--) {
put_task_struct(next_task);
next_task = task;
@@ -373,6 +394,158 @@ static void push_rt_tasks(struct rq *rq)
;
}
+static int pull_rt_task(struct rq *this_rq)
+{
+ struct task_struct *next;
+ struct task_struct *p;
+ struct rq *src_rq;
+ cpumask_t *rto_cpumask;
+ int this_cpu = this_rq-cpu;
+ int cpu;
+ int ret = 0;
+
+ assert_spin_locked(this_rq-lock);
+
+ /*
+* If cpusets are used, and we have overlapping
+* run queue cpusets, then this algorithm may not catch all.
+* This is just the price you pay
