The local callbacks processing checks if some callbacks need acceleration. Keep that behaviour under nocb lock protection when rcu_core() executes concurrently with GP/CB kthreads.
Inspired-by: Paul E. McKenney <paul...@kernel.org> Signed-off-by: Frederic Weisbecker <frede...@kernel.org> Cc: Paul E. McKenney <paul...@kernel.org> Cc: Josh Triplett <j...@joshtriplett.org> Cc: Steven Rostedt <rost...@goodmis.org> Cc: Mathieu Desnoyers <mathieu.desnoy...@efficios.com> Cc: Lai Jiangshan <jiangshan...@gmail.com> Cc: Joel Fernandes <j...@joelfernandes.org> Cc: Neeraj Upadhyay <neer...@codeaurora.org> Cc: Thomas Gleixner <t...@linutronix.de> Cc: Boqun Feng <boqun.f...@gmail.com> --- kernel/rcu/tree.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index cc141337f3f9..b24d60a52f34 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2690,7 +2690,6 @@ static __latent_entropy void rcu_core(void) unsigned long flags; struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); struct rcu_node *rnp = rdp->mynode; - const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist); const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist); if (cpu_is_offline(smp_processor_id())) @@ -2711,11 +2710,11 @@ static __latent_entropy void rcu_core(void) /* No grace period and unregistered callbacks? */ if (!rcu_gp_in_progress() && - rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) { - local_irq_save(flags); + rcu_segcblist_is_enabled(&rdp->cblist) && do_batch) { + rcu_nocb_lock_irqsave(rdp, flags); if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) rcu_accelerate_cbs_unlocked(rnp, rdp); - local_irq_restore(flags); + rcu_nocb_unlock_irqrestore(rdp, flags); } rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); -- 2.25.1