[PATCH tip/core/rcu 09/12] rcu: Push lock release to rcu_start_gp()'s callers
From: "Paul E. McKenney"
If CPUs are to give prior notice of needed grace periods, it will be
necessary to invoke rcu_start_gp() without dropping the root rcu_node
structure's ->lock. This commit takes a second step in this direction
by moving the release of this lock to rcu_start_gp()'s callers.
Signed-off-by: Paul E. McKenney
Signed-off-by: Paul E. McKenney
---
kernel/rcutree.c| 24 ++--
kernel/rcutree_plugin.h |5 ++---
2 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 2c6a931..0d53295 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1487,16 +1487,14 @@ static int __noreturn rcu_gp_kthread(void *arg)
/*
* Start a new RCU grace period if warranted, re-initializing the hierarchy
* in preparation for detecting the next grace period. The caller must hold
- * the root node's ->lock, which is released before return. Hard irqs must
- * be disabled.
+ * the root node's ->lock and hard irqs must be disabled.
*
* Note that it is legal for a dying CPU (which is marked as offline) to
* invoke this function. This can happen when the dying CPU reports its
* quiescent state.
*/
static void
-rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
- __releases(rcu_get_root(rsp)->lock)
+rcu_start_gp(struct rcu_state *rsp)
{
struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
struct rcu_node *rnp = rcu_get_root(rsp);
@@ -1510,15 +1508,13 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
*/
rcu_advance_cbs(rsp, rnp, rdp);
- if (!rsp->gp_kthread ||
- !cpu_needs_another_gp(rsp, rdp)) {
+ if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) {
/*
* Either we have not yet spawned the grace-period
* task, this CPU does not need another grace period,
* or a grace period is already in progress.
* Either way, don't start a new grace period.
*/
- raw_spin_unlock_irqrestore(>lock, flags);
return;
}
rsp->gp_flags = RCU_GP_FLAG_INIT;
@@ -1528,15 +1524,14 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
/* Wake up rcu_gp_kthread() to start the grace period. */
wake_up(>gp_wq);
- raw_spin_unlock_irqrestore(>lock, flags);
}
/*
* Report a full set of quiescent states to the specified rcu_state
* data structure. This involves cleaning up after the prior grace
* period and letting rcu_start_gp() start up the next grace period
- * if one is needed. Note that the caller must hold rnp->lock, as
- * required by rcu_start_gp(), which will release it.
+ * if one is needed. Note that the caller must hold rnp->lock, which
+ * is released before return.
*/
static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
__releases(rcu_get_root(rsp)->lock)
@@ -2134,7 +2129,8 @@ __rcu_process_callbacks(struct rcu_state *rsp)
local_irq_save(flags);
if (cpu_needs_another_gp(rsp, rdp)) {
raw_spin_lock(_get_root(rsp)->lock); /* irqs disabled. */
- rcu_start_gp(rsp, flags); /* releases above lock */
+ rcu_start_gp(rsp);
+ raw_spin_unlock_irqrestore(_get_root(rsp)->lock, flags);
} else {
local_irq_restore(flags);
}
@@ -2214,11 +2210,11 @@ static void __call_rcu_core(struct rcu_state *rsp,
struct rcu_data *rdp,
/* Start a new grace period if one not already started. */
if (!rcu_gp_in_progress(rsp)) {
- unsigned long nestflag;
struct rcu_node *rnp_root = rcu_get_root(rsp);
- raw_spin_lock_irqsave(_root->lock, nestflag);
- rcu_start_gp(rsp, nestflag); /* rlses rnp_root->lock */
+ raw_spin_lock(_root->lock);
+ rcu_start_gp(rsp);
+ raw_spin_unlock(_root->lock);
} else {
/* Give the grace period a kick. */
rdp->blimit = LONG_MAX;
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index bcd8268..7a66312 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -2220,7 +2220,6 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
unsigned long c;
bool d;
unsigned long flags;
- unsigned long flags1;
struct rcu_node *rnp = rdp->mynode;
struct rcu_node *rnp_root = rcu_get_root(rdp->rsp);
@@ -2282,8 +2281,8 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
c, rnp->level,
rnp->grplo, rnp->grphi,
"Startedroot");
- local_save_flags(flags1);
-
[PATCH tip/core/rcu 09/12] rcu: Push lock release to rcu_start_gp()'s callers
From: Paul E. McKenney paul.mcken...@linaro.org
If CPUs are to give prior notice of needed grace periods, it will be
necessary to invoke rcu_start_gp() without dropping the root rcu_node
structure's -lock. This commit takes a second step in this direction
by moving the release of this lock to rcu_start_gp()'s callers.
Signed-off-by: Paul E. McKenney paul.mcken...@linaro.org
Signed-off-by: Paul E. McKenney paul...@linux.vnet.ibm.com
---
kernel/rcutree.c| 24 ++--
kernel/rcutree_plugin.h |5 ++---
2 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 2c6a931..0d53295 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1487,16 +1487,14 @@ static int __noreturn rcu_gp_kthread(void *arg)
/*
* Start a new RCU grace period if warranted, re-initializing the hierarchy
* in preparation for detecting the next grace period. The caller must hold
- * the root node's -lock, which is released before return. Hard irqs must
- * be disabled.
+ * the root node's -lock and hard irqs must be disabled.
*
* Note that it is legal for a dying CPU (which is marked as offline) to
* invoke this function. This can happen when the dying CPU reports its
* quiescent state.
*/
static void
-rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
- __releases(rcu_get_root(rsp)-lock)
+rcu_start_gp(struct rcu_state *rsp)
{
struct rcu_data *rdp = this_cpu_ptr(rsp-rda);
struct rcu_node *rnp = rcu_get_root(rsp);
@@ -1510,15 +1508,13 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
*/
rcu_advance_cbs(rsp, rnp, rdp);
- if (!rsp-gp_kthread ||
- !cpu_needs_another_gp(rsp, rdp)) {
+ if (!rsp-gp_kthread || !cpu_needs_another_gp(rsp, rdp)) {
/*
* Either we have not yet spawned the grace-period
* task, this CPU does not need another grace period,
* or a grace period is already in progress.
* Either way, don't start a new grace period.
*/
- raw_spin_unlock_irqrestore(rnp-lock, flags);
return;
}
rsp-gp_flags = RCU_GP_FLAG_INIT;
@@ -1528,15 +1524,14 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
/* Wake up rcu_gp_kthread() to start the grace period. */
wake_up(rsp-gp_wq);
- raw_spin_unlock_irqrestore(rnp-lock, flags);
}
/*
* Report a full set of quiescent states to the specified rcu_state
* data structure. This involves cleaning up after the prior grace
* period and letting rcu_start_gp() start up the next grace period
- * if one is needed. Note that the caller must hold rnp-lock, as
- * required by rcu_start_gp(), which will release it.
+ * if one is needed. Note that the caller must hold rnp-lock, which
+ * is released before return.
*/
static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
__releases(rcu_get_root(rsp)-lock)
@@ -2134,7 +2129,8 @@ __rcu_process_callbacks(struct rcu_state *rsp)
local_irq_save(flags);
if (cpu_needs_another_gp(rsp, rdp)) {
raw_spin_lock(rcu_get_root(rsp)-lock); /* irqs disabled. */
- rcu_start_gp(rsp, flags); /* releases above lock */
+ rcu_start_gp(rsp);
+ raw_spin_unlock_irqrestore(rcu_get_root(rsp)-lock, flags);
} else {
local_irq_restore(flags);
}
@@ -2214,11 +2210,11 @@ static void __call_rcu_core(struct rcu_state *rsp,
struct rcu_data *rdp,
/* Start a new grace period if one not already started. */
if (!rcu_gp_in_progress(rsp)) {
- unsigned long nestflag;
struct rcu_node *rnp_root = rcu_get_root(rsp);
- raw_spin_lock_irqsave(rnp_root-lock, nestflag);
- rcu_start_gp(rsp, nestflag); /* rlses rnp_root-lock */
+ raw_spin_lock(rnp_root-lock);
+ rcu_start_gp(rsp);
+ raw_spin_unlock(rnp_root-lock);
} else {
/* Give the grace period a kick. */
rdp-blimit = LONG_MAX;
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index bcd8268..7a66312 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -2220,7 +2220,6 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
unsigned long c;
bool d;
unsigned long flags;
- unsigned long flags1;
struct rcu_node *rnp = rdp-mynode;
struct rcu_node *rnp_root = rcu_get_root(rdp-rsp);
@@ -2282,8 +2281,8 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
c, rnp-level,
rnp-grplo, rnp-grphi,
