Re: [PATCH] sched: Allow per-cpu kernel threads to run on online && !active

[Thomas Gleixner]

On Tue, 1 Mar 2016, Peter Zijlstra wrote:
> Tested-by: Thomas Gleixner <t...@linutronix.de>

Works nicely especially when we move the set_cpu_active() calls as the last
state in the state machine. See patch below.

Thanks,

        tglx

8<--------------------------

Subject: cpu/hotplug: Handle cpu active as last state
From: Thomas Gleixner <t...@linutronix.de>
Date: Thu, 03 Mar 2016 12:33:37 +0100

We want to make sure that everything is initialized before we allow scheduling
of arbitrary work on a upcoming CPU and we don't want to have no more random
work on it when we shut it down.

Now that the scheduler handles this nicely via the cpu_active_mask we can put
set_cpu_active() as last action when a cpu is brought up and as first action
when it goes down.

Signed-off-by: Thomas Gleixner <t...@linutronix.de>
Cc: Peter Zijlstra <pet...@infradead.org>
---
 include/linux/cpuhotplug.h |    1 +
 kernel/cpu.c               |   19 +++++++++++++++++--
 kernel/sched/core.c        |   18 ------------------
 3 files changed, 18 insertions(+), 20 deletions(-)

--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -16,6 +16,7 @@ enum cpuhp_state {
        CPUHP_AP_NOTIFY_ONLINE,
        CPUHP_AP_ONLINE_DYN,
        CPUHP_AP_ONLINE_DYN_END         = CPUHP_AP_ONLINE_DYN + 30,
+       CPUHP_AP_ACTIVE,
        CPUHP_ONLINE,
 };
 
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -904,8 +904,6 @@ void cpuhp_online_idle(enum cpuhp_state
 
        st->state = CPUHP_AP_ONLINE_IDLE;
 
-       /* The cpu is marked online, set it active now */
-       set_cpu_active(cpu, true);
        /* Unpark the stopper thread and the hotplug thread of this cpu */
        stop_machine_unpark(cpu);
        kthread_unpark(st->thread);
@@ -917,6 +915,18 @@ void cpuhp_online_idle(enum cpuhp_state
                complete(&st->done);
 }
 
+static int cpu_activate(unsigned int cpu)
+{
+       set_cpu_active(cpu, true);
+       return 0;
+}
+
+static int cpu_deactivate(unsigned int cpu)
+{
+       set_cpu_active(cpu, false);
+       return 0;
+}
+
 /* Requires cpu_add_remove_lock to be held */
 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 {
@@ -1213,6 +1223,11 @@ static struct cpuhp_step cpuhp_ap_states
                .startup                = notify_online,
                .teardown               = notify_down_prepare,
        },
+       [CPUHP_AP_ACTIVE] = {
+               .name                   = "active",
+               .startup                = cpu_activate,
+               .teardown               = cpu_deactivate,
+       },
 #endif
        [CPUHP_ONLINE] = {
                .name                   = "online",
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5726,23 +5726,6 @@ static int sched_cpu_active(struct notif
        case CPU_STARTING:
                set_cpu_rq_start_time();
                return NOTIFY_OK;
-
-       case CPU_DOWN_FAILED:
-               set_cpu_active(cpu, true);
-               return NOTIFY_OK;
-
-       default:
-               return NOTIFY_DONE;
-       }
-}
-
-static int sched_cpu_inactive(struct notifier_block *nfb,
-                                       unsigned long action, void *hcpu)
-{
-       switch (action & ~CPU_TASKS_FROZEN) {
-       case CPU_DOWN_PREPARE:
-               set_cpu_active((long)hcpu, false);
-               return NOTIFY_OK;
        default:
                return NOTIFY_DONE;
        }
@@ -5761,7 +5744,6 @@ static int __init migration_init(void)
 
        /* Register cpu active notifiers */
        cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
-       cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
 
        return 0;
 }