[PATCH tip/core/rcu 12/40] rcu: Pull rcu_qs_ctr into rcu_dynticks structure

[Paul E. McKenney]

The rcu_qs_ctr variable is yet another isolated per-CPU variable,
so this commit pulls it into the pre-existing rcu_dynticks per-CPU
structure.

Signed-off-by: Paul E. McKenney 
---
 .../RCU/Design/Data-Structures/Data-Structures.html   | 12 ++--
 kernel/rcu/tree.c | 15 ++-
 kernel/rcu/tree.h |  3 ++-
 kernel/rcu/tree_trace.c   |  4 +---
 4 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/Documentation/RCU/Design/Data-Structures/Data-Structures.html 
b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
index bf7f266e..3d0311657533 100644
--- a/Documentation/RCU/Design/Data-Structures/Data-Structures.html
+++ b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
@@ -1105,6 +1105,7 @@ Its fields are as follows:
   2   int dynticks_nmi_nesting;
   3   atomic_t dynticks;
   4   int rcu_sched_qs_mask;
+  5   unsigned long rcu_qs_ctr;
 
 
 The -dynticks_nesting field counts the
@@ -1123,12 +1124,19 @@ CPU's transitions to and from dyntick-idle mode, so 
that this counter
 has an even value when the CPU is in dyntick-idle mode and an odd
 value otherwise.
 
-Finally, the  -rcu_sched_qs_mask field is used
+The -rcu_sched_qs_mask field is used
 to record the fact that the RCU core code would really like to
-see a quiescent state from the corresponding CPU.
+see a quiescent state from the corresponding CPU, so much so that
+it is willing to call for heavy-weight dyntick-counter operations.
 This flag is checked by RCU's context-switch and cond_resched()
 code, which provide a momentary idle sojourn in response.
 
+Finally the -rcu_qs_ctr field is used to record
+quiescent states from cond_resched().
+Because cond_resched() can execute quite frequently, this
+must be quite lightweight, as in a non-atomic increment of this
+per-CPU field.
+
 
 
 Quick Quiz:
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 315647d4e4cd..be40ebb4f0bf 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -436,9 +436,6 @@ bool rcu_eqs_special_set(int cpu)
return true;
 }
 
-DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
-EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
-
 /*
  * Let the RCU core know that this CPU has gone through the scheduler,
  * which is a quiescent state.  This is called when the need for a
@@ -552,7 +549,7 @@ void rcu_all_qs(void)
do_nocb_deferred_wakeup(this_cpu_ptr(rsp->rda));
preempt_enable();
}
-   this_cpu_inc(rcu_qs_ctr);
+   this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
barrier(); /* Avoid RCU read-side critical sections leaking up. */
 }
 EXPORT_SYMBOL_GPL(rcu_all_qs);
@@ -1325,7 +1322,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 */
rnp = rdp->mynode;
if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
-   READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
+   READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, 
rdp->cpu) &&
READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
return 1;
@@ -2034,7 +2031,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, 
struct rcu_node *rnp,
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
need_gp = !!(rnp->qsmask & rdp->grpmask);
rdp->cpu_no_qs.b.norm = need_gp;
-   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
+   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
rdp->core_needs_qs = need_gp;
zero_cpu_stall_ticks(rdp);
WRITE_ONCE(rdp->gpwrap, false);
@@ -2632,7 +2629,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct 
rcu_data *rdp)
 * within the current grace period.
 */
rdp->cpu_no_qs.b.norm = true;   /* need qs for new gp. */
-   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
+   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
return;
}
@@ -3630,7 +3627,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct 
rcu_data *rdp)
/* Is the RCU core waiting for a quiescent state from this CPU? */
if (rcu_scheduler_fully_active &&
rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
-   rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
+   rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_dynticks.rcu_qs_ctr)) {
rdp->n_rp_core_needs_qs++;
} else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
rdp->n_rp_report_qs++;
@@ -3943,7 +3940,7 @@ rcu_init_percpu_data(int cpu, struct 

[PATCH tip/core/rcu 12/40] rcu: Pull rcu_qs_ctr into rcu_dynticks structure

[Paul E. McKenney]

The rcu_qs_ctr variable is yet another isolated per-CPU variable,
so this commit pulls it into the pre-existing rcu_dynticks per-CPU
structure.

Signed-off-by: Paul E. McKenney 
---
 .../RCU/Design/Data-Structures/Data-Structures.html   | 12 ++--
 kernel/rcu/tree.c | 15 ++-
 kernel/rcu/tree.h |  3 ++-
 kernel/rcu/tree_trace.c   |  4 +---
 4 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/Documentation/RCU/Design/Data-Structures/Data-Structures.html 
b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
index bf7f266e..3d0311657533 100644
--- a/Documentation/RCU/Design/Data-Structures/Data-Structures.html
+++ b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
@@ -1105,6 +1105,7 @@ Its fields are as follows:
   2   int dynticks_nmi_nesting;
   3   atomic_t dynticks;
   4   int rcu_sched_qs_mask;
+  5   unsigned long rcu_qs_ctr;
 
 
 The -dynticks_nesting field counts the
@@ -1123,12 +1124,19 @@ CPU's transitions to and from dyntick-idle mode, so 
that this counter
 has an even value when the CPU is in dyntick-idle mode and an odd
 value otherwise.
 
-Finally, the  -rcu_sched_qs_mask field is used
+The -rcu_sched_qs_mask field is used
 to record the fact that the RCU core code would really like to
-see a quiescent state from the corresponding CPU.
+see a quiescent state from the corresponding CPU, so much so that
+it is willing to call for heavy-weight dyntick-counter operations.
 This flag is checked by RCU's context-switch and cond_resched()
 code, which provide a momentary idle sojourn in response.
 
+Finally the -rcu_qs_ctr field is used to record
+quiescent states from cond_resched().
+Because cond_resched() can execute quite frequently, this
+must be quite lightweight, as in a non-atomic increment of this
+per-CPU field.
+
 
 
 Quick Quiz:
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 315647d4e4cd..be40ebb4f0bf 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -436,9 +436,6 @@ bool rcu_eqs_special_set(int cpu)
return true;
 }
 
-DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
-EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
-
 /*
  * Let the RCU core know that this CPU has gone through the scheduler,
  * which is a quiescent state.  This is called when the need for a
@@ -552,7 +549,7 @@ void rcu_all_qs(void)
do_nocb_deferred_wakeup(this_cpu_ptr(rsp->rda));
preempt_enable();
}
-   this_cpu_inc(rcu_qs_ctr);
+   this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
barrier(); /* Avoid RCU read-side critical sections leaking up. */
 }
 EXPORT_SYMBOL_GPL(rcu_all_qs);
@@ -1325,7 +1322,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 */
rnp = rdp->mynode;
if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
-   READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
+   READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, 
rdp->cpu) &&
READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
return 1;
@@ -2034,7 +2031,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, 
struct rcu_node *rnp,
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
need_gp = !!(rnp->qsmask & rdp->grpmask);
rdp->cpu_no_qs.b.norm = need_gp;
-   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
+   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
rdp->core_needs_qs = need_gp;
zero_cpu_stall_ticks(rdp);
WRITE_ONCE(rdp->gpwrap, false);
@@ -2632,7 +2629,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct 
rcu_data *rdp)
 * within the current grace period.
 */
rdp->cpu_no_qs.b.norm = true;   /* need qs for new gp. */
-   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
+   rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
return;
}
@@ -3630,7 +3627,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct 
rcu_data *rdp)
/* Is the RCU core waiting for a quiescent state from this CPU? */
if (rcu_scheduler_fully_active &&
rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
-   rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
+   rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_dynticks.rcu_qs_ctr)) {
rdp->n_rp_core_needs_qs++;
} else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
rdp->n_rp_report_qs++;
@@ -3943,7 +3940,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)