Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG to avoid sending IPIs to idle CPUs.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]> --- kernel/sched/core.c | 10 ++++++++++ kernel/sched/idle.c | 1 + kernel/sched/sched.h | 2 ++ kernel/smp.c | 16 +++++++++++++++- 4 files changed, 28 insertions(+), 1 deletion(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info) sched_ttwu_pending(); } +void send_call_function_single_ipi(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (!set_nr_if_polling(rq->idle)) + arch_send_call_function_single_ipi(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); +} + /* * Queue a task on the target CPUs wake_list and wake the CPU via IPI if * necessary. The wakee CPU on receipt of the IPI will queue the task --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -289,6 +289,7 @@ static void do_idle(void) */ smp_mb__after_atomic(); + flush_smp_call_function_from_idle(); sched_ttwu_pending(); schedule_idle(); --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1506,6 +1506,8 @@ static inline void unregister_sched_doma } #endif +extern void flush_smp_call_function_from_idle(void); + #else static inline void sched_ttwu_pending(void) { } --- a/kernel/smp.c +++ b/kernel/smp.c @@ -135,6 +135,8 @@ static __always_inline void csd_unlock(c static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); +extern void send_call_function_single_ipi(int cpu); + /* * Insert a previously allocated call_single_data_t element * for execution on the given CPU. data must already have @@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, * equipped to do the right thing... */ if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) - arch_send_call_function_single_ipi(cpu); + send_call_function_single_ipi(cpu); return 0; } @@ -278,6 +280,18 @@ static void flush_smp_call_function_queu } } +void flush_smp_call_function_from_idle(void) +{ + unsigned long flags; + + if (llist_empty(this_cpu_ptr(&call_single_queue))) + return; + + local_irq_save(flags); + flush_smp_call_function_queue(true); + local_irq_restore(flags); +} + /* * smp_call_function_single - Run a function on a specific CPU * @func: The function to run. This must be fast and non-blocking.
