Add a new cpuhp_offline_cb() API that allows us to offline a set of CPUs one-by-one, run the given callback function and then bring those CPUs back online again while inhibiting any concurrent CPU hotplug operations from happening.
This new API can be used to enable runtime adjustment of nohz_full and isolcpus boot command line options. A new cpuhp_offline_cb_mode flag is also added to signal that the system is in this offline callback transient state so that some hotplug operations can be optimized out if we choose to. Signed-off-by: Waiman Long <[email protected]> --- include/linux/cpuhplock.h | 9 +++++ kernel/cpu.c | 70 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/include/linux/cpuhplock.h b/include/linux/cpuhplock.h index 286b3ab92e15..37637baa32eb 100644 --- a/include/linux/cpuhplock.h +++ b/include/linux/cpuhplock.h @@ -9,7 +9,9 @@ #include <linux/cleanup.h> #include <linux/errno.h> +#include <linux/cpumask_types.h> +typedef int (*cpuhp_cb_t)(void *arg); struct device; extern int lockdep_is_cpus_held(void); @@ -29,6 +31,8 @@ void clear_tasks_mm_cpumask(int cpu); int remove_cpu(unsigned int cpu); int cpu_device_down(struct device *dev); void smp_shutdown_nonboot_cpus(unsigned int primary_cpu); +int cpuhp_offline_cb(struct cpumask *mask, cpuhp_cb_t func, void *arg); +extern bool cpuhp_offline_cb_mode; #else /* CONFIG_HOTPLUG_CPU */ @@ -43,6 +47,11 @@ static inline void cpu_hotplug_disable(void) { } static inline void cpu_hotplug_enable(void) { } static inline int remove_cpu(unsigned int cpu) { return -EPERM; } static inline void smp_shutdown_nonboot_cpus(unsigned int primary_cpu) { } +static inline int cpuhp_offline_cb(struct cpumask *mask, cpuhp_cb_t func, void *arg) +{ + return -EPERM; +} +#define cpuhp_offline_cb_mode false #endif /* !CONFIG_HOTPLUG_CPU */ DEFINE_LOCK_GUARD_0(cpus_read_lock, cpus_read_lock(), cpus_read_unlock()) diff --git a/kernel/cpu.c b/kernel/cpu.c index 0d02b5d7a7ba..9b32f742cd1d 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1520,6 +1520,76 @@ int remove_cpu(unsigned int cpu) } EXPORT_SYMBOL_GPL(remove_cpu); +bool cpuhp_offline_cb_mode; + +/** + * cpuhp_offline_cb - offline CPUs, invoke callback function & online CPUs afterward + * @mask: A mask of CPUs to be taken offline and then online + * @func: A callback function to be invoked while the given CPUs are offline + * @arg: Argument to be passed back to the callback function + * + * Return: 0 if successful, an error code otherwise + */ +int cpuhp_offline_cb(struct cpumask *mask, cpuhp_cb_t func, void *arg) +{ + int off_cpu, on_cpu, ret, ret2 = 0; + + if (WARN_ON_ONCE(cpumask_empty(mask) || + !cpumask_subset(mask, cpu_online_mask))) + return -EINVAL; + + pr_debug("%s: begin (CPU list = %*pbl)\n", __func__, cpumask_pr_args(mask)); + lock_device_hotplug(); + cpuhp_offline_cb_mode = true; + /* + * If all offline operations succeed, off_cpu should become nr_cpu_ids. + */ + for_each_cpu(off_cpu, mask) { + ret = device_offline(get_cpu_device(off_cpu)); + if (unlikely(ret)) + break; + } + if (!ret) + ret = func(arg); + + /* Bring previously offline CPUs back online */ + for_each_cpu(on_cpu, mask) { + int retries = 0; + + if (on_cpu == off_cpu) + break; + +retry: + ret2 = device_online(get_cpu_device(on_cpu)); + + /* + * With the unlikely event that CPU hotplug is disabled while + * this operation is in progress, we will need to wait a bit + * for hotplug to hopefully be re-enabled again. If not, print + * a warning and return the error. + * + * cpu_hotplug_disabled is supposed to be accessed while + * holding the cpu_add_remove_lock mutex. So we need to + * use the data_race() macro to access it here. + */ + while ((ret2 == -EBUSY) && data_race(cpu_hotplug_disabled) && + (++retries <= 5)) { + msleep(20); + if (!data_race(cpu_hotplug_disabled)) + goto retry; + } + if (ret2) { + pr_warn("%s: Failed to bring CPU %d back online!\n", + __func__, on_cpu); + break; + } + } + cpuhp_offline_cb_mode = false; + unlock_device_hotplug(); + pr_debug("%s: end\n", __func__); + return ret ? ret : ret2; +} + void smp_shutdown_nonboot_cpus(unsigned int primary_cpu) { unsigned int cpu; -- 2.53.0
