When a cpuset isolated partition first requests kernel-noise isolation,
it needs to offload RCU callbacks from the affected CPUs. The existing
rcu_nocb_cpu_offload() requires rcu_nocbs= or nohz_full= at boot so
that rcu_organize_nocb_kthreads() has already set rdp->nocb_gp_rdp for
each CPU. Without a boot parameter, nocb_gp_rdp is NULL and offload
fails immediately.
Introduce lazy nocb initialization so that the first call into the
isolation path triggers the one-time setup automatically:
rcu_nocb_lazy_init() - allocates rcu_nocb_mask and calls
rcu_organize_nocb_kthreads() (now without __init) to set
nocb_gp_rdp for every possible CPU. Uses a dedicated mutex
for serialization with a fast-path read of nocb_is_setup.
Registers the CONFIG_RCU_LAZY shrinker on this first-ever
setup too: rcu_init_nohz() only does so when nocb_is_setup was
already true at boot, so the lazy-callback reclaim path would
otherwise never exist on a system with no rcu_nocbs=/nohz_full=
boot parameter. Factor the registration into
rcu_nocb_register_lazy_shrinker(), shared with rcu_init_nohz().
rcu_nocb_cpu_isolate() - exported entry point called per-CPU
while the CPU is offline. Calls rcu_nocb_lazy_init() for the
one-time setup, spawns the GP and CB kthreads via the existing
rcu_spawn_cpu_nocb_kthread(), then finalizes offload through
rcu_nocb_cpu_offload(). Adding the CPU to the GP kthread's
nocb_head_rdp list is handled by nocb_gp_toggle_rdp() in the
GP kthread, so rcu_organize_nocb_kthreads() can run with an
empty mask.
Remove __init from rcu_organize_nocb_kthreads() to allow this
runtime call path; the function itself has no __initdata dependencies.
Signed-off-by: Qiliang Yuan <[email protected]>
---
include/linux/rcupdate.h | 2 ++
kernel/rcu/tree.h | 2 +-
kernel/rcu/tree_nocb.h | 81 ++++++++++++++++++++++++++++++++++++++++--------
3 files changed, 71 insertions(+), 14 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 44c07a66edfff..1425f04ceb4e9 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -158,6 +158,7 @@ static __always_inline void rcu_irq_work_resched(void) { }
void rcu_init_nohz(void);
int rcu_nocb_cpu_offload(int cpu);
int rcu_nocb_cpu_deoffload(int cpu);
+int rcu_nocb_cpu_isolate(int cpu);
void rcu_nocb_flush_deferred_wakeup(void);
#define RCU_NOCB_LOCKDEP_WARN(c, s) RCU_LOCKDEP_WARN(c, s)
@@ -167,6 +168,7 @@ void rcu_nocb_flush_deferred_wakeup(void);
static inline void rcu_init_nohz(void) { }
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
+static inline int rcu_nocb_cpu_isolate(int cpu) { return -EINVAL; }
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
#define RCU_NOCB_LOCKDEP_WARN(c, s)
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index eedfa43059e80..60bcf52817f3e 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -522,7 +522,7 @@ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
unsigned long flags);
static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);
#ifdef CONFIG_RCU_NOCB_CPU
-static void __init rcu_organize_nocb_kthreads(void);
+static void rcu_organize_nocb_kthreads(void);
/*
* Disable IRQs before checking offloaded state so that local
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index 19bb42672baf8..99f3e4cec34e3 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -1344,12 +1344,30 @@ lazy_rcu_shrink_scan(struct shrinker *shrink, struct
shrink_control *sc)
}
#endif // #ifdef CONFIG_RCU_LAZY
+#ifdef CONFIG_RCU_LAZY
+static void rcu_nocb_register_lazy_shrinker(void)
+{
+ struct shrinker *lazy_rcu_shrinker = shrinker_alloc(0, "rcu-lazy");
+
+ if (!lazy_rcu_shrinker) {
+ pr_err("Failed to allocate lazy_rcu shrinker!\n");
+ return;
+ }
+
+ lazy_rcu_shrinker->count_objects = lazy_rcu_shrink_count;
+ lazy_rcu_shrinker->scan_objects = lazy_rcu_shrink_scan;
+
+ shrinker_register(lazy_rcu_shrinker);
+}
+#else
+static void rcu_nocb_register_lazy_shrinker(void) { }
+#endif // #ifdef CONFIG_RCU_LAZY
+
void __init rcu_init_nohz(void)
{
int cpu;
struct rcu_data *rdp;
const struct cpumask *cpumask = NULL;
- struct shrinker * __maybe_unused lazy_rcu_shrinker;
#if defined(CONFIG_NO_HZ_FULL)
if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask))
@@ -1375,17 +1393,7 @@ void __init rcu_init_nohz(void)
if (!rcu_state.nocb_is_setup)
return;
-#ifdef CONFIG_RCU_LAZY
- lazy_rcu_shrinker = shrinker_alloc(0, "rcu-lazy");
- if (!lazy_rcu_shrinker) {
- pr_err("Failed to allocate lazy_rcu shrinker!\n");
- } else {
- lazy_rcu_shrinker->count_objects = lazy_rcu_shrink_count;
- lazy_rcu_shrinker->scan_objects = lazy_rcu_shrink_scan;
-
- shrinker_register(lazy_rcu_shrinker);
- }
-#endif // #ifdef CONFIG_RCU_LAZY
+ rcu_nocb_register_lazy_shrinker();
if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or
'isolcpus=' contains nonexistent CPUs.\n");
@@ -1409,6 +1417,53 @@ void __init rcu_init_nohz(void)
rcu_organize_nocb_kthreads();
}
+static DEFINE_MUTEX(rcu_nocb_lazy_mutex);
+
+/*
+ * Lazily initialize nocb infrastructure on the first call. Allocates
+ * rcu_nocb_mask and sets nocb_gp_rdp for every possible CPU so that
+ * rcu_nocb_cpu_isolate() can offload callbacks without rcu_nocbs= at boot.
+ */
+static noinline int rcu_nocb_lazy_init(void)
+{
+ if (rcu_state.nocb_is_setup)
+ return 0;
+
+ mutex_lock(&rcu_nocb_lazy_mutex);
+ if (!rcu_state.nocb_is_setup) {
+ if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
+ mutex_unlock(&rcu_nocb_lazy_mutex);
+ return -ENOMEM;
+ }
+ rcu_organize_nocb_kthreads();
+ /*
+ * Boot-time rcu_init_nohz() only registers the lazy-callback
+ * shrinker when nocb_is_setup was already true at boot; this
+ * is the first time it is set, so register it here instead.
+ */
+ rcu_nocb_register_lazy_shrinker();
+ rcu_state.nocb_is_setup = true;
+ }
+ mutex_unlock(&rcu_nocb_lazy_mutex);
+ return 0;
+}
+
+/*
+ * Offload RCU callbacks for a CPU entering a kernel-noise isolated partition.
+ * @cpu must be offline. Lazily initializes nocb infrastructure on first use.
+ */
+int rcu_nocb_cpu_isolate(int cpu)
+{
+ int ret;
+
+ ret = rcu_nocb_lazy_init();
+ if (ret)
+ return ret;
+ rcu_spawn_cpu_nocb_kthread(cpu);
+ return rcu_nocb_cpu_offload(cpu);
+}
+EXPORT_SYMBOL_GPL(rcu_nocb_cpu_isolate);
+
/* Initialize per-rcu_data variables for no-CBs CPUs. */
static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
{
@@ -1501,7 +1556,7 @@ module_param(rcu_nocb_gp_stride, int, 0444);
/*
* Initialize GP-CB relationships for all no-CBs CPU.
*/
-static void __init rcu_organize_nocb_kthreads(void)
+static void rcu_organize_nocb_kthreads(void)
{
int cpu;
bool firsttime = true;
--
2.43.0
