Make use of the provided rcu_nocb_cpu_offload()/rcu_nocb_cpu_deoffload() APIs to enable RCU NO-CB CPU offloading of newly isolated CPUs and deoffloading of de-isolated CPUs.
Also add a new rcu_nocbs_enabled() helper function to determine if RCU NO-CB CPU offloading can be done. As nohz_full can now be specified without any CPU list, drop the test for cpumask_empty(tick_nohz_full_mask) in rcu_init_nohz(). The RCU NO-CB CPU offloading feature can only used if either "rcs_nocbs" or the "nohz_full" boot command parameters are used so that the proper RCU NO-CB resources are properly initialized at boot time. Signed-off-by: Waiman Long <[email protected]> --- include/linux/rcupdate.h | 2 ++ kernel/cgroup/cpuset.c | 14 ++++++++++++++ kernel/rcu/tree_nocb.h | 7 ++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 120536f4c6eb..642b80a4f071 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -140,6 +140,7 @@ void rcu_init_nohz(void); int rcu_nocb_cpu_offload(int cpu); int rcu_nocb_cpu_deoffload(int cpu); void rcu_nocb_flush_deferred_wakeup(void); +bool rcu_nocbs_enabled(void); #define RCU_NOCB_LOCKDEP_WARN(c, s) RCU_LOCKDEP_WARN(c, s) @@ -149,6 +150,7 @@ static inline void rcu_init_nohz(void) { } static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; } static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; } static inline void rcu_nocb_flush_deferred_wakeup(void) { } +static inline bool rcu_nocbs_enabled(void) { return false; } #define RCU_NOCB_LOCKDEP_WARN(c, s) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45c82c18bec4..de9cb92a0fc7 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1416,6 +1416,20 @@ static int do_housekeeping_exclude_cpumask(void *arg __maybe_unused) } #endif + if (rcu_nocbs_enabled()) { + /* + * Enable RCU NO-CB CPU offloading/deoffloading for the affected CPUs + */ + for_each_cpu(cpu, isolcpus_update_state.cpus) { + if (cpumask_test_cpu(cpu, icpus)) + ret = rcu_nocb_cpu_offload(cpu); + else + ret = rcu_nocb_cpu_deoffload(cpu); + if (WARN_ON_ONCE(ret)) + break; + } + } + if (icpus != isolated_cpus) kfree(icpus); return ret; diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index e6cd56603cad..4d49a745b871 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1293,7 +1293,7 @@ void __init rcu_init_nohz(void) struct shrinker * __maybe_unused lazy_rcu_shrinker; #if defined(CONFIG_NO_HZ_FULL) - if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) + if (tick_nohz_full_running) cpumask = tick_nohz_full_mask; #endif @@ -1365,6 +1365,11 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) mutex_init(&rdp->nocb_gp_kthread_mutex); } +bool rcu_nocbs_enabled(void) +{ + return !!rcu_state.nocb_is_setup; +} + /* * If the specified CPU is a no-CBs CPU that does not already have its * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread -- 2.50.0
