The following commit has been merged into the core/rcu branch of tip:
Commit-ID: 32aa2f4170d22f0b9fcb75ab05679ab122fae373
Gitweb:
https://git.kernel.org/tip/32aa2f4170d22f0b9fcb75ab05679ab122fae373
Author: Frederic Weisbecker <frede...@kernel.org>
AuthorDate: Fri, 13 Nov 2020 13:13:27 +01:00
Committer: Paul E. McKenney <paul...@kernel.org>
CommitterDate: Wed, 06 Jan 2021 16:24:59 -08:00
rcu/nocb: Process batch locally as long as offloading isn't complete
This commit makes sure to process the callbacks locally (via either
RCU_SOFTIRQ or the rcuc kthread) whenever the segcblist isn't entirely
offloaded. This ensures that callbacks are invoked one way or another
while a CPU is in the middle of a toggle operation.
Cc: Josh Triplett <j...@joshtriplett.org>
Cc: Steven Rostedt <rost...@goodmis.org>
Cc: Mathieu Desnoyers <mathieu.desnoy...@efficios.com>
Cc: Lai Jiangshan <jiangshan...@gmail.com>
Cc: Joel Fernandes <j...@joelfernandes.org>
Cc: Neeraj Upadhyay <neer...@codeaurora.org>
Cc: Thomas Gleixner <t...@linutronix.de>
Inspired-by: Paul E. McKenney <paul...@kernel.org>
Tested-by: Boqun Feng <boqun.f...@gmail.com>
Signed-off-by: Frederic Weisbecker <frede...@kernel.org>
Signed-off-by: Paul E. McKenney <paul...@kernel.org>
---
kernel/rcu/rcu_segcblist.h | 12 ++++++++++++
kernel/rcu/tree.c | 3 ++-
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h
index 28c9a52..afad6fc 100644
--- a/kernel/rcu/rcu_segcblist.h
+++ b/kernel/rcu/rcu_segcblist.h
@@ -95,6 +95,18 @@ static inline bool rcu_segcblist_is_offloaded(struct
rcu_segcblist *rsclp)
return false;
}
+static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist
*rsclp)
+{
+ int flags = SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |
SEGCBLIST_OFFLOADED;
+
+ if (IS_ENABLED(CONFIG_RCU_NOCB_CPU)) {
+ if ((rsclp->flags & flags) == flags)
+ return true;
+ }
+
+ return false;
+}
+
/*
* Are all segments following the specified segment of the specified
* rcu_segcblist structure empty of callbacks? (The specified
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 4ef59a5..ec14c01 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2700,6 +2700,7 @@ static __latent_entropy void rcu_core(void)
struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
struct rcu_node *rnp = rdp->mynode;
const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist);
+ const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist);
if (cpu_is_offline(smp_processor_id()))
return;
@@ -2729,7 +2730,7 @@ static __latent_entropy void rcu_core(void)
rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
/* If there are callbacks ready, invoke them. */
- if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
+ if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) &&
likely(READ_ONCE(rcu_scheduler_fully_active)))
rcu_do_batch(rdp);
