Add support for scaling atomic SRCU grace periods using srcu_read_lock_atomic()/srcu_read_unlock_atomic() and synchronize_srcu_atomic().
Suggested-by: Paul E. McKenney <[email protected]> Signed-off-by: Kunwu Chan <[email protected]> --- Changes since v1: - Rename srcu_atomic_ctlp to srcua and reuse srcu_ctlp for SRCU operations, as suggested by Paul. - Reuse the existing SRCU cleanup and completed helpers by initializing srcu_ctlp in srcu_atomic_scale_init(). - Add .stats using srcu_scale_stats(). - Use "srcua" to match the existing "srcu" and "srcud" scale type names. - Use synchronize_srcu_atomic() for both normal and expedited scale tests. v1: https://lore.kernel.org/all/[email protected]/ Tested with kvm.sh --torture rcuscale scale_type=srcua on x86_64 and arm64, alongside gp_exp=1 and PROVE_LOCKING=y: no warnings or lockdep splats. --- kernel/rcu/rcuscale.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 1097ec15879c..b172882824ac 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -293,6 +293,44 @@ static struct rcu_scale_ops srcud_ops = { .name = "srcud" }; +static struct srcu_struct srcua; + +static int srcu_scale_atomic_read_lock(void) +{ + return srcu_read_lock_atomic(srcu_ctlp); +} + +static void srcu_scale_atomic_read_unlock(int idx) +{ + srcu_read_unlock_atomic(srcu_ctlp, idx); +} + +static void srcu_scale_atomic_synchronize(void) +{ + synchronize_srcu_atomic(srcu_ctlp); +} + +static void srcu_atomic_scale_init(void) +{ + srcu_ctlp = &srcua; + init_srcu_struct_atomic(srcu_ctlp); +} + +static struct rcu_scale_ops srcua_ops = { + .ptype = SRCU_FLAVOR, + .init = srcu_atomic_scale_init, + .cleanup = srcu_sync_scale_cleanup, + .readlock = srcu_scale_atomic_read_lock, + .readunlock = srcu_scale_atomic_read_unlock, + .get_gp_seq = srcu_scale_completed, + .gp_diff = rcu_seq_diff, + .exp_completed = srcu_scale_completed, + .sync = srcu_scale_atomic_synchronize, + .exp_sync = srcu_scale_atomic_synchronize, + .stats = srcu_scale_stats, + .name = "srcua" +}; + #ifdef CONFIG_TASKS_RCU /* @@ -1110,7 +1148,8 @@ rcu_scale_init(void) long i; long j; static struct rcu_scale_ops *scale_ops[] = { - &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS + &rcu_ops, &srcu_ops, &srcua_ops, &srcud_ops, + TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS }; if (!torture_init_begin(scale_type, verbose)) base-commit: d21906b0aa1e9573cdb5e7acaca44966b9d1dcd2 -- 2.43.0
