On Tue, Jan 17, 2017 at 06:45:25PM -0800, Paul E. McKenney wrote: > This commit is the third step towards full abstraction of all accesses > to the ->dynticks counter, implementing the previously open-coded atomic > add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and > the same but with exit checks in a new rcu_dynticks_eqs_exit() function. > This abstraction will ease changes to the ->dynticks counter operation. > > Note that this commit gets rid of the smp_mb__before_atomic() and the > smp_mb__after_atomic() calls that were previously present. The reason > that this is OK from a memory-ordering perspective is that the atomic > operation is now atomic_add_return(), which, as a value-returning atomic, > guarantees full ordering. > > Signed-off-by: Paul E. McKenney <[email protected]>
A couple of issues with RCU_TRACE usage noted below. With those fixed: Reviewed-by: Josh Triplett <[email protected]>. > kernel/rcu/tree.c | 88 > +++++++++++++++++++++++++++++++++++++++---------------- > 1 file changed, 62 insertions(+), 26 deletions(-) > > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c > index 805d55ee0b2a..70b01e1983e6 100644 > --- a/kernel/rcu/tree.c > +++ b/kernel/rcu/tree.c > @@ -282,6 +282,61 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) > = { > }; > > /* > + * Record entry into an extended quiescent state. This is only to be > + * called when not already in an extended quiescent state. > + */ > +static void rcu_dynticks_eqs_enter(void) > +{ > + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); > + int special; > + > + /* > + * CPUs seeing atomic_inc_return() must see prior RCU read-side > + * critical sections, and we also must force ordering with the > + * next idle sojourn. > + */ > + special = atomic_inc_return(&rdtp->dynticks); > + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && special & 0x1); > +} > + > +/* > + * Record exit from an extended quiescent state. This is only to be > + * called from an extended quiescent state. > + */ > +static void rcu_dynticks_eqs_exit(void) > +{ > + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); > + int special; > + > + /* > + * CPUs seeing atomic_inc_return() must see prior idle sojourns, > + * and we also must force ordering with the next RCU read-side > + * critical section. > + */ > + special = atomic_inc_return(&rdtp->dynticks); > + WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(special & 0x1)); > +} > + > +/* > + * Reset the current CPU's ->dynticks counter to indicate that the > + * newly onlined CPU is no longer in an extended quiescent state. > + * This will either leave the counter unchanged, or increment it > + * to the next non-quiescent value. > + * > + * The non-atomic test/increment sequence works because the upper bits > + * of the ->dynticks counter are manipulated only by the corresponding CPU, > + * or when the corresponding CPU is offline. > + */ > +static void rcu_dynticks_eqs_online(void) > +{ > + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); > + > + if (atomic_read(&rdtp->dynticks) & 0x1) > + return; > + atomic_add(0x1, &rdtp->dynticks); > +} > + > +/* > * Snapshot the ->dynticks counter with full ordering so as to allow > * stable comparison of this counter with past and future snapshots. > */ > @@ -693,7 +748,7 @@ static void rcu_eqs_enter_common(long long oldval, bool > user) > { > struct rcu_state *rsp; > struct rcu_data *rdp; > - struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); > + RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks)); RCU_TRACE doesn't act like a statement, and doesn't take a semicolon; it just acts like a shorthand for an #ifdef. The semicolon belongs inside the macro parentheses, since it goes with the declaration. Otherwise, this compiles to the empty statement ';' without tracing. Among other things, that would lead to surprises for anyone who added a subsequent declaration, because ';' followed by a declaration will produce a -Wdeclaration-after-statement warning. > trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting); > if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && > @@ -712,12 +767,7 @@ static void rcu_eqs_enter_common(long long oldval, bool > user) > do_nocb_deferred_wakeup(rdp); > } > rcu_prepare_for_idle(); > - /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ > - smp_mb__before_atomic(); /* See above. */ > - atomic_inc(&rdtp->dynticks); > - smp_mb__after_atomic(); /* Force ordering with next sojourn. */ > - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && > - atomic_read(&rdtp->dynticks) & 0x1); > + rcu_dynticks_eqs_enter(); > rcu_dynticks_task_enter(); > > /* > @@ -846,15 +896,10 @@ void rcu_irq_exit_irqson(void) > */ > static void rcu_eqs_exit_common(long long oldval, int user) > { > - struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); > + RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks)); As above. > rcu_dynticks_task_exit(); > - smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */ > - atomic_inc(&rdtp->dynticks); > - /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ > - smp_mb__after_atomic(); /* See above. */ > - WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && > - !(atomic_read(&rdtp->dynticks) & 0x1)); > + rcu_dynticks_eqs_exit(); > rcu_cleanup_after_idle(); > trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting); > if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && > @@ -1001,11 +1046,7 @@ void rcu_nmi_enter(void) > * period (observation due to Andy Lutomirski). > */ > if (!(atomic_read(&rdtp->dynticks) & 0x1)) { > - smp_mb__before_atomic(); /* Force delay from prior write. */ > - atomic_inc(&rdtp->dynticks); > - /* atomic_inc() before later RCU read-side crit sects */ > - smp_mb__after_atomic(); /* See above. */ > - WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); > + rcu_dynticks_eqs_exit(); > incby = 1; > } > rdtp->dynticks_nmi_nesting += incby; > @@ -1043,11 +1084,7 @@ void rcu_nmi_exit(void) > > /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ > rdtp->dynticks_nmi_nesting = 0; > - /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ > - smp_mb__before_atomic(); /* See above. */ > - atomic_inc(&rdtp->dynticks); > - smp_mb__after_atomic(); /* Force delay to next write. */ > - WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); > + rcu_dynticks_eqs_enter(); > } > > /** > @@ -3800,8 +3837,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) > init_callback_list(rdp); /* Re-enable callbacks on this CPU. */ > rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; > rcu_sysidle_init_percpu_data(rdp->dynticks); > - atomic_set(&rdp->dynticks->dynticks, > - (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1); > + rcu_dynticks_eqs_online(); > raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ > > /* > -- > 2.5.2 >
