The scheduller code is written with the assumption, that rq->curr task can't
be already put. For example, in sched_move_task() we check for
running = task_current(rq, tsk);
and call put_prev_task() if "running" is true.
When we're unlocking rq->lock in trigger_cpulimit_balance(), the task has
already been put, so concurrent cpu_cgroup_attach_task()->sched_move_task()
puts it one more time.
https://jira.sw.ru/browse/PSBM-35082
Signed-off-by: Kirill Tkhai <[email protected]>
---
kernel/sched/core.c | 8 ++++++++
kernel/sched/fair.c | 34 +++++++++++++++-------------------
kernel/sched/sched.h | 2 ++
3 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 30f39a25..bb6ee5a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2209,6 +2209,14 @@ static inline void pre_schedule(struct rq *rq, struct
task_struct *prev)
{
if (prev->sched_class->pre_schedule)
prev->sched_class->pre_schedule(rq, prev);
+
+ if (rq->active_balance == ACTIVE_BALANCE_CPULIMIT) {
+ raw_spin_unlock(&rq->lock);
+ stop_one_cpu_nowait(rq->cpu,
+ cpulimit_balance_cpu_stop, rq,
+ &rq->active_balance_work);
+ raw_spin_lock(&rq->lock);
+ }
}
/* rq->lock is NOT held, but preemption is disabled */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 167d0f6..32188a8 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5059,8 +5059,6 @@ static struct task_struct *pick_next_task_fair(struct rq
*rq)
}
#if defined(CONFIG_SMP) && defined(CONFIG_CFS_CPULIMIT)
-static int cpulimit_balance_cpu_stop(void *data);
-
static inline void trigger_cpulimit_balance(struct task_struct *p)
{
struct rq *this_rq;
@@ -5068,18 +5066,16 @@ static inline void trigger_cpulimit_balance(struct
task_struct *p)
int this_cpu, cpu, target_cpu = -1;
struct sched_domain *sd;
- if (!p->se.on_rq)
- return;
-
this_rq = rq_of(cfs_rq_of(&p->se));
this_cpu = cpu_of(this_rq);
+ if (!p->se.on_rq || this_rq->active_balance)
+ return;
+
cfs_rq = top_cfs_rq_of(&p->se);
if (check_cpulimit_spread(cfs_rq, this_cpu) >= 0)
return;
- raw_spin_unlock(&this_rq->lock);
-
rcu_read_lock();
for_each_domain(this_cpu, sd) {
if (!(sd->flags & SD_LOAD_BALANCE))
@@ -5096,17 +5092,10 @@ static inline void trigger_cpulimit_balance(struct
task_struct *p)
unlock:
rcu_read_unlock();
- raw_spin_lock(&this_rq->lock);
if (target_cpu >= 0) {
- if (!this_rq->active_balance) {
- this_rq->active_balance = 1;
- this_rq->push_cpu = target_cpu;
- raw_spin_unlock(&this_rq->lock);
- stop_one_cpu_nowait(this_cpu,
- cpulimit_balance_cpu_stop, this_rq,
- &this_rq->active_balance_work);
- raw_spin_lock(&this_rq->lock);
- }
+ this_rq->active_balance = ACTIVE_BALANCE_CPULIMIT;
+ this_rq->push_cpu = target_cpu;
+ resched_task(this_rq->curr);
}
}
#else
@@ -5777,7 +5766,7 @@ static int do_cpulimit_balance(struct lb_env *env)
return pushed;
}
-static int cpulimit_balance_cpu_stop(void *data)
+int cpulimit_balance_cpu_stop(void *data)
{
struct rq *rq = data;
int cpu = cpu_of(rq);
@@ -5787,7 +5776,8 @@ static int cpulimit_balance_cpu_stop(void *data)
raw_spin_lock_irq(&rq->lock);
- if (unlikely(cpu != smp_processor_id() || !rq->active_balance))
+ if (unlikely(cpu != smp_processor_id() || !rq->active_balance ||
+ !cpu_online(target_cpu)))
goto out_unlock;
if (unlikely(!rq->nr_running))
@@ -7269,6 +7259,11 @@ static int active_load_balance_cpu_stop(void *data)
return 0;
}
+static void pre_schedule_fair(struct rq *rq, struct task_struct *prev)
+{
+ trigger_cpulimit_balance(prev);
+}
+
#ifdef CONFIG_NO_HZ_COMMON
/*
* idle load balancing details
@@ -8171,6 +8166,7 @@ const struct sched_class fair_sched_class = {
.rq_offline = rq_offline_fair,
.task_waking = task_waking_fair,
+ .pre_schedule = pre_schedule_fair,
#endif
.set_curr_task = set_curr_task_fair,
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 44c5ad6..ae01198 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -509,6 +509,7 @@ struct rq {
/* For active balancing */
int post_schedule;
int active_balance;
+#define ACTIVE_BALANCE_CPULIMIT 0x10000
int push_cpu;
struct cpu_stop_work active_balance_work;
/* cpu of this runqueue: */
@@ -1111,6 +1112,7 @@ extern const struct sched_class idle_sched_class;
extern void update_group_power(struct sched_domain *sd, int cpu);
+extern int cpulimit_balance_cpu_stop(void *data);
extern void trigger_load_balance(struct rq *rq, int cpu);
extern void idle_balance(int this_cpu, struct rq *this_rq);
_______________________________________________
Devel mailing list
[email protected]
https://lists.openvz.org/mailman/listinfo/devel
