[tip: sched/core] rcu: Pull deferred rcuog wake up to rcu_eqs_enter() callers
The following commit has been merged into the sched/core branch of tip: Commit-ID: 54b7429efffc99e845ba9381bee3244f012a06c2 Gitweb: https://git.kernel.org/tip/54b7429efffc99e845ba9381bee3244f012a06c2 Author:Frederic Weisbecker AuthorDate:Mon, 01 Feb 2021 00:05:44 +01:00 Committer: Ingo Molnar CommitterDate: Wed, 17 Feb 2021 14:12:42 +01:00 rcu: Pull deferred rcuog wake up to rcu_eqs_enter() callers Deferred wakeup of rcuog kthreads upon RCU idle mode entry is going to be handled differently whether initiated by idle, user or guest. Prepare with pulling that control up to rcu_eqs_enter() callers. Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: sta...@vger.kernel.org Link: https://lkml.kernel.org/r/20210131230548.32970-2-frede...@kernel.org --- kernel/rcu/tree.c | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 40e5e3d..63032e5 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -644,7 +644,6 @@ static noinstr void rcu_eqs_enter(bool user) trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(>dynticks)); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rdp = this_cpu_ptr(_data); - do_nocb_deferred_wakeup(rdp); rcu_prepare_for_idle(); rcu_preempt_deferred_qs(current); @@ -672,7 +671,10 @@ static noinstr void rcu_eqs_enter(bool user) */ void rcu_idle_enter(void) { + struct rcu_data *rdp = this_cpu_ptr(_data); + lockdep_assert_irqs_disabled(); + do_nocb_deferred_wakeup(rdp); rcu_eqs_enter(false); } EXPORT_SYMBOL_GPL(rcu_idle_enter); @@ -691,7 +693,14 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); */ noinstr void rcu_user_enter(void) { + struct rcu_data *rdp = this_cpu_ptr(_data); + lockdep_assert_irqs_disabled(); + + instrumentation_begin(); + do_nocb_deferred_wakeup(rdp); + instrumentation_end(); + rcu_eqs_enter(true); } #endif /* CONFIG_NO_HZ_FULL */
[tip: sched/core] rcu: Pull deferred rcuog wake up to rcu_eqs_enter() callers
The following commit has been merged into the sched/core branch of tip: Commit-ID: e4234f21d2ea7674bcc1aeaca9d382b50ca1efec Gitweb: https://git.kernel.org/tip/e4234f21d2ea7674bcc1aeaca9d382b50ca1efec Author:Frederic Weisbecker AuthorDate:Mon, 01 Feb 2021 00:05:44 +01:00 Committer: Peter Zijlstra CommitterDate: Wed, 10 Feb 2021 14:44:49 +01:00 rcu: Pull deferred rcuog wake up to rcu_eqs_enter() callers Deferred wakeup of rcuog kthreads upon RCU idle mode entry is going to be handled differently whether initiated by idle, user or guest. Prepare with pulling that control up to rcu_eqs_enter() callers. Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Cc: sta...@vger.kernel.org Link: https://lkml.kernel.org/r/20210131230548.32970-2-frede...@kernel.org --- kernel/rcu/tree.c | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 40e5e3d..63032e5 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -644,7 +644,6 @@ static noinstr void rcu_eqs_enter(bool user) trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(>dynticks)); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); rdp = this_cpu_ptr(_data); - do_nocb_deferred_wakeup(rdp); rcu_prepare_for_idle(); rcu_preempt_deferred_qs(current); @@ -672,7 +671,10 @@ static noinstr void rcu_eqs_enter(bool user) */ void rcu_idle_enter(void) { + struct rcu_data *rdp = this_cpu_ptr(_data); + lockdep_assert_irqs_disabled(); + do_nocb_deferred_wakeup(rdp); rcu_eqs_enter(false); } EXPORT_SYMBOL_GPL(rcu_idle_enter); @@ -691,7 +693,14 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); */ noinstr void rcu_user_enter(void) { + struct rcu_data *rdp = this_cpu_ptr(_data); + lockdep_assert_irqs_disabled(); + + instrumentation_begin(); + do_nocb_deferred_wakeup(rdp); + instrumentation_end(); + rcu_eqs_enter(true); } #endif /* CONFIG_NO_HZ_FULL */