[PATCH tip/core/rcu 05/12] rcu: Accelerate RCU callbacks at grace-period end
From: "Paul E. McKenney"
Now that callback acceleration is idempotent, it is safe to accelerate
callbacks during grace-period cleanup on any CPUs that the kthread happens
to be running on. This commit therefore propagates the completion
of the grace period to the per-CPU data structures, and also adds an
rcu_advance_cbs() just before the cpu_needs_another_gp() check in order
to reduce false-positive grace periods.
Signed-off-by: Paul E. McKenney
---
kernel/rcutree.c | 21 +
1 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 074cb2d..2015bce 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1396,6 +1396,9 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
rcu_for_each_node_breadth_first(rsp, rnp) {
raw_spin_lock_irq(>lock);
rnp->completed = rsp->gpnum;
+ rdp = this_cpu_ptr(rsp->rda);
+ if (rnp == rdp->mynode)
+ __rcu_process_gp_end(rsp, rnp, rdp);
nocb += rcu_nocb_gp_cleanup(rsp, rnp);
raw_spin_unlock_irq(>lock);
cond_resched();
@@ -1408,6 +1411,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
trace_rcu_grace_period(rsp->name, rsp->completed, "end");
rsp->fqs_state = RCU_GP_IDLE;
rdp = this_cpu_ptr(rsp->rda);
+ rcu_advance_cbs(rsp, rnp, rdp); /* Reduce false positives below. */
if (cpu_needs_another_gp(rsp, rdp))
rsp->gp_flags = 1;
raw_spin_unlock_irq(>lock);
@@ -1497,6 +1501,15 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
struct rcu_node *rnp = rcu_get_root(rsp);
+ /*
+* If there is no grace period in progress right now, any
+* callbacks we have up to this point will be satisfied by the
+* next grace period. Also, advancing the callbacks reduces the
+* probability of false positives from cpu_needs_another_gp()
+* resulting in pointless grace periods. So, advance callbacks!
+*/
+ rcu_advance_cbs(rsp, rnp, rdp);
+
if (!rsp->gp_kthread ||
!cpu_needs_another_gp(rsp, rdp)) {
/*
@@ -1509,14 +1522,6 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
return;
}
- /*
-* Because there is no grace period in progress right now,
-* any callbacks we have up to this point will be satisfied
-* by the next grace period. So this is a good place to
-* assign a grace period number to recently posted callbacks.
-*/
- rcu_accelerate_cbs(rsp, rnp, rdp);
-
rsp->gp_flags = RCU_GP_FLAG_INIT;
raw_spin_unlock(>lock); /* Interrupts remain disabled. */
--
1.7.8
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[PATCH tip/core/rcu 05/12] rcu: Accelerate RCU callbacks at grace-period end
From: Paul E. McKenney paul.mcken...@linaro.org
Now that callback acceleration is idempotent, it is safe to accelerate
callbacks during grace-period cleanup on any CPUs that the kthread happens
to be running on. This commit therefore propagates the completion
of the grace period to the per-CPU data structures, and also adds an
rcu_advance_cbs() just before the cpu_needs_another_gp() check in order
to reduce false-positive grace periods.
Signed-off-by: Paul E. McKenney paul...@linux.vnet.ibm.com
---
kernel/rcutree.c | 21 +
1 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 074cb2d..2015bce 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1396,6 +1396,9 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
rcu_for_each_node_breadth_first(rsp, rnp) {
raw_spin_lock_irq(rnp-lock);
rnp-completed = rsp-gpnum;
+ rdp = this_cpu_ptr(rsp-rda);
+ if (rnp == rdp-mynode)
+ __rcu_process_gp_end(rsp, rnp, rdp);
nocb += rcu_nocb_gp_cleanup(rsp, rnp);
raw_spin_unlock_irq(rnp-lock);
cond_resched();
@@ -1408,6 +1411,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
trace_rcu_grace_period(rsp-name, rsp-completed, end);
rsp-fqs_state = RCU_GP_IDLE;
rdp = this_cpu_ptr(rsp-rda);
+ rcu_advance_cbs(rsp, rnp, rdp); /* Reduce false positives below. */
if (cpu_needs_another_gp(rsp, rdp))
rsp-gp_flags = 1;
raw_spin_unlock_irq(rnp-lock);
@@ -1497,6 +1501,15 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
struct rcu_data *rdp = this_cpu_ptr(rsp-rda);
struct rcu_node *rnp = rcu_get_root(rsp);
+ /*
+* If there is no grace period in progress right now, any
+* callbacks we have up to this point will be satisfied by the
+* next grace period. Also, advancing the callbacks reduces the
+* probability of false positives from cpu_needs_another_gp()
+* resulting in pointless grace periods. So, advance callbacks!
+*/
+ rcu_advance_cbs(rsp, rnp, rdp);
+
if (!rsp-gp_kthread ||
!cpu_needs_another_gp(rsp, rdp)) {
/*
@@ -1509,14 +1522,6 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
return;
}
- /*
-* Because there is no grace period in progress right now,
-* any callbacks we have up to this point will be satisfied
-* by the next grace period. So this is a good place to
-* assign a grace period number to recently posted callbacks.
-*/
- rcu_accelerate_cbs(rsp, rnp, rdp);
-
rsp-gp_flags = RCU_GP_FLAG_INIT;
raw_spin_unlock(rnp-lock); /* Interrupts remain disabled. */
--
1.7.8
--
To unsubscribe from this list: send the line unsubscribe linux-kernel in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
