The following commit has been merged into the core/rcu branch of tip: Commit-ID: 32aa2f4170d22f0b9fcb75ab05679ab122fae373 Gitweb: https://git.kernel.org/tip/32aa2f4170d22f0b9fcb75ab05679ab122fae373 Author: Frederic Weisbecker <frede...@kernel.org> AuthorDate: Fri, 13 Nov 2020 13:13:27 +01:00 Committer: Paul E. McKenney <paul...@kernel.org> CommitterDate: Wed, 06 Jan 2021 16:24:59 -08:00
rcu/nocb: Process batch locally as long as offloading isn't complete This commit makes sure to process the callbacks locally (via either RCU_SOFTIRQ or the rcuc kthread) whenever the segcblist isn't entirely offloaded. This ensures that callbacks are invoked one way or another while a CPU is in the middle of a toggle operation. Cc: Josh Triplett <j...@joshtriplett.org> Cc: Steven Rostedt <rost...@goodmis.org> Cc: Mathieu Desnoyers <mathieu.desnoy...@efficios.com> Cc: Lai Jiangshan <jiangshan...@gmail.com> Cc: Joel Fernandes <j...@joelfernandes.org> Cc: Neeraj Upadhyay <neer...@codeaurora.org> Cc: Thomas Gleixner <t...@linutronix.de> Inspired-by: Paul E. McKenney <paul...@kernel.org> Tested-by: Boqun Feng <boqun.f...@gmail.com> Signed-off-by: Frederic Weisbecker <frede...@kernel.org> Signed-off-by: Paul E. McKenney <paul...@kernel.org> --- kernel/rcu/rcu_segcblist.h | 12 ++++++++++++ kernel/rcu/tree.c | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 28c9a52..afad6fc 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -95,6 +95,18 @@ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) return false; } +static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp) +{ + int flags = SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | SEGCBLIST_OFFLOADED; + + if (IS_ENABLED(CONFIG_RCU_NOCB_CPU)) { + if ((rsclp->flags & flags) == flags) + return true; + } + + return false; +} + /* * Are all segments following the specified segment of the specified * rcu_segcblist structure empty of callbacks? (The specified diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 4ef59a5..ec14c01 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2700,6 +2700,7 @@ static __latent_entropy void rcu_core(void) struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); struct rcu_node *rnp = rdp->mynode; const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist); + const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist); if (cpu_is_offline(smp_processor_id())) return; @@ -2729,7 +2730,7 @@ static __latent_entropy void rcu_core(void) rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); /* If there are callbacks ready, invoke them. */ - if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) && + if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) && likely(READ_ONCE(rcu_scheduler_fully_active))) rcu_do_batch(rdp);