Rt and deadline use their separate percpu mask to do SMP scheduling.
All the operations related to the mask are under the protection of
some spin_lock_irqsave(), hence we can safely use a shared one.
The patch introduced "sched_pp_shared_mask"(pp means Push/Pull which
are main users of the mask), removed all the init_sched_rt(dl)_class()
stuff and initialized the shared mask directly in sched_init() instead.
We can do this, because at the sched_init() stage there is only one task
(to be "idle") and with irq disabled, no schedule can happen, also the
"cpu_possible_mask" was already initiated.
After that we again safely removed the NULL cpumask judgement from
find_lowest_rq() and find_later_rq(), because "sched_pp_shared_mask"
was surely initialized before the first SMP scheduling happens.
Inspired-by: Peter Zijlstra <pet...@infradead.org>
Signed-off-by: Xunlei Pang <xlp...@redhat.com>
---
kernel/sched/core.c | 13 ++++++++++---
kernel/sched/deadline.c | 18 ++----------------
kernel/sched/rt.c | 19 ++-----------------
kernel/sched/sched.h | 6 ++++--
4 files changed, 18 insertions(+), 38 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index a533566..dbc22ab 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -93,6 +93,11 @@
DEFINE_MUTEX(sched_domains_mutex);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+#ifdef CONFIG_SMP
+/* Used by Push/Pull scheduling, and shared by rt and deadline. */
+DEFINE_PER_CPU(cpumask_var_t, sched_pp_shared_mask);
+#endif
+
static void update_rq_clock_task(struct rq *rq, s64 delta);
void update_rq_clock(struct rq *rq)
@@ -7187,9 +7192,6 @@ void __init sched_init_smp(void)
BUG();
sched_init_granularity();
free_cpumask_var(non_isolated_cpus);
-
- init_sched_rt_class();
- init_sched_dl_class();
}
#else
void __init sched_init_smp(void)
@@ -7389,6 +7391,11 @@ void __init sched_init(void)
zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
idle_thread_set_boot_cpu();
set_cpu_rq_start_time();
+
+ for_each_possible_cpu(i) {
+ zalloc_cpumask_var_node(&per_cpu(sched_pp_shared_mask, i),
+ GFP_KERNEL, cpu_to_node(i));
+ }
#endif
init_sched_fair_class();
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index e564c88..169d40d 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1269,19 +1269,13 @@ next_node:
return NULL;
}
-static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl);
-
static int find_later_rq(struct task_struct *task)
{
struct sched_domain *sd;
- struct cpumask *later_mask =
this_cpu_cpumask_var_ptr(local_cpu_mask_dl);
+ struct cpumask *later_mask;
int this_cpu = smp_processor_id();
int best_cpu, cpu = task_cpu(task);
- /* Make sure the mask is initialized first */
- if (unlikely(!later_mask))
- return -1;
-
if (task->nr_cpus_allowed == 1)
return -1;
@@ -1289,6 +1283,7 @@ static int find_later_rq(struct task_struct *task)
* We have to consider system topology and task affinity
* first, then we can look for a suitable cpu.
*/
+ later_mask = this_cpu_cpumask_var_ptr(sched_pp_shared_mask);
best_cpu = cpudl_find(&task_rq(task)->rd->cpudl,
task, later_mask);
if (best_cpu == -1)
@@ -1675,15 +1670,6 @@ static void rq_offline_dl(struct rq *rq)
cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu);
}
-void __init init_sched_dl_class(void)
-{
- unsigned int i;
-
- for_each_possible_cpu(i)
- zalloc_cpumask_var_node(&per_cpu(local_cpu_mask_dl, i),
- GFP_KERNEL, cpu_to_node(i));
-}
-
#endif /* CONFIG_SMP */
static void switched_from_dl(struct rq *rq, struct task_struct *p)
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 5624713..ecfc83d 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1612,22 +1612,17 @@ static struct task_struct
*pick_highest_pushable_task(struct rq *rq, int cpu)
return NULL;
}
-static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
-
static int find_lowest_rq(struct task_struct *task)
{
struct sched_domain *sd;
- struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
+ struct cpumask *lowest_mask;
int this_cpu = smp_processor_id();
int cpu = task_cpu(task);
- /* Make sure the mask is initialized first */
- if (unlikely(!lowest_mask))
- return -1;
-
if (task->nr_cpus_allowed == 1)
return -1; /* No other targets possible */
+ lowest_mask = this_cpu_cpumask_var_ptr(sched_pp_shared_mask);
if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
return -1; /* No targets found */
@@ -2164,16 +2159,6 @@ static void switched_from_rt(struct rq *rq, struct
task_struct *p)
queue_pull_task(rq);
}
-
-void __init init_sched_rt_class(void)
-{
- unsigned int i;
-
- for_each_possible_cpu(i) {
- zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
- GFP_KERNEL, cpu_to_node(i));
- }
-}
#endif /* CONFIG_SMP */
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e6d4a3f..e880b37 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -715,6 +715,10 @@ static inline int cpu_of(struct rq *rq)
DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+#ifdef CONFIG_SMP
+DECLARE_PER_CPU(cpumask_var_t, sched_pp_shared_mask);
+#endif
+
#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
#define this_rq() this_cpu_ptr(&runqueues)
#define task_rq(p) cpu_rq(task_cpu(p))
@@ -1296,8 +1300,6 @@ extern void sysrq_sched_debug_show(void);
extern void sched_init_granularity(void);
extern void update_max_interval(void);
-extern void init_sched_dl_class(void);
-extern void init_sched_rt_class(void);
extern void init_sched_fair_class(void);
extern void resched_curr(struct rq *rq);
--
1.8.3.1
