[PATCH tip/core/rcu 08/14] rcu: Make RCU_FAST_NO_HZ take advantage of numbered callbacks
From: "Paul E. McKenney"
Because RCU callbacks are now associated with the number of the grace
period that they must wait for, CPUs can now take advance callbacks
corresponding to grace periods that ended while a given CPU was in
dyntick-idle mode. This eliminates the need to try forcing the RCU
state machine while entering idle, thus reducing the CPU intensiveness
of RCU_FAST_NO_HZ, which should increase its energy efficiency.
Signed-off-by: Paul E. McKenney
Signed-off-by: Paul E. McKenney
---
include/linux/rcupdate.h |1 +
kernel/rcutree.c | 28 +++--
kernel/rcutree.h | 12 +--
kernel/rcutree_plugin.h | 350 +++---
kernel/rcutree_trace.c |2 -
5 files changed, 131 insertions(+), 262 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 275aa3f..4b37b50 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -73,6 +73,7 @@ extern void do_trace_rcu_torture_read(char *rcutorturename,
#define UINT_CMP_LT(a, b) (UINT_MAX / 2 < (a) - (b))
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
+#define ulong2long(a) (*(long *)(&(a)))
/* Exported common interfaces */
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 392c977..4b8d91c 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2678,19 +2678,27 @@ static int rcu_pending(int cpu)
}
/*
- * Check to see if any future RCU-related work will need to be done
- * by the current CPU, even if none need be done immediately, returning
- * 1 if so.
+ * Return true if the specified CPU has any callback. If all_lazy is
+ * non-NULL, store an indication of whether all callbacks are lazy.
+ * (If there are no callbacks, all of them are deemed to be lazy.)
*/
-static int rcu_cpu_has_callbacks(int cpu)
+static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
{
+ bool al = true;
+ bool hc = false;
+ struct rcu_data *rdp;
struct rcu_state *rsp;
- /* RCU callbacks either ready or pending? */
- for_each_rcu_flavor(rsp)
- if (per_cpu_ptr(rsp->rda, cpu)->nxtlist)
- return 1;
- return 0;
+ for_each_rcu_flavor(rsp) {
+ rdp = per_cpu_ptr(rsp->rda, cpu);
+ if (rdp->qlen != rdp->qlen_lazy)
+ al = false;
+ if (rdp->nxtlist)
+ hc = true;
+ }
+ if (all_lazy)
+ *all_lazy = al;
+ return hc;
}
/*
@@ -2912,7 +2920,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int
preemptible)
rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
atomic_set(>dynticks->dynticks,
(atomic_read(>dynticks->dynticks) & ~0x1) + 1);
- rcu_prepare_for_idle_init(cpu);
raw_spin_unlock(>lock);/* irqs remain disabled. */
/* Add CPU to rcu_node bitmasks. */
@@ -2986,7 +2993,6 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block
*self,
*/
for_each_rcu_flavor(rsp)
rcu_cleanup_dying_cpu(rsp);
- rcu_cleanup_after_idle(cpu);
break;
case CPU_DEAD:
case CPU_DEAD_FROZEN:
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index c865117..282b1d7 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -88,18 +88,13 @@ struct rcu_dynticks {
int dynticks_nmi_nesting; /* Track NMI nesting level. */
atomic_t dynticks; /* Even value for idle, else odd. */
#ifdef CONFIG_RCU_FAST_NO_HZ
- int dyntick_drain; /* Prepare-for-idle state variable. */
- unsigned long dyntick_holdoff;
- /* No retries for the jiffy of failure. */
- struct timer_list idle_gp_timer;
- /* Wake up CPU sleeping with callbacks. */
- unsigned long idle_gp_timer_expires;
- /* When to wake up CPU (for repost). */
- bool idle_first_pass; /* First pass of attempt to go idle? */
+ bool all_lazy; /* Are all CPU's CBs lazy? */
unsigned long nonlazy_posted;
/* # times non-lazy CBs posted to CPU. */
unsigned long nonlazy_posted_snap;
/* idle-period nonlazy_posted snapshot. */
+ unsigned long last_accelerate;
+ /* Last jiffy CBs were accelerated. */
int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
#ifdef CONFIG_RCU_USER_QS
@@ -530,7 +525,6 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct
rcu_state *rsp,
struct rcu_node *rnp);
#endif /* #ifdef CONFIG_RCU_BOOST */
static void __cpuinit rcu_prepare_kthreads(int
[PATCH tip/core/rcu 08/14] rcu: Make RCU_FAST_NO_HZ take advantage of numbered callbacks
From: Paul E. McKenney paul.mcken...@linaro.org
Because RCU callbacks are now associated with the number of the grace
period that they must wait for, CPUs can now take advance callbacks
corresponding to grace periods that ended while a given CPU was in
dyntick-idle mode. This eliminates the need to try forcing the RCU
state machine while entering idle, thus reducing the CPU intensiveness
of RCU_FAST_NO_HZ, which should increase its energy efficiency.
Signed-off-by: Paul E. McKenney paul.mcken...@linaro.org
Signed-off-by: Paul E. McKenney paul...@linux.vnet.ibm.com
---
include/linux/rcupdate.h |1 +
kernel/rcutree.c | 28 +++--
kernel/rcutree.h | 12 +--
kernel/rcutree_plugin.h | 350 +++---
kernel/rcutree_trace.c |2 -
5 files changed, 131 insertions(+), 262 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 275aa3f..4b37b50 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -73,6 +73,7 @@ extern void do_trace_rcu_torture_read(char *rcutorturename,
#define UINT_CMP_LT(a, b) (UINT_MAX / 2 (a) - (b))
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 = (a) - (b))
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 (a) - (b))
+#define ulong2long(a) (*(long *)((a)))
/* Exported common interfaces */
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 392c977..4b8d91c 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2678,19 +2678,27 @@ static int rcu_pending(int cpu)
}
/*
- * Check to see if any future RCU-related work will need to be done
- * by the current CPU, even if none need be done immediately, returning
- * 1 if so.
+ * Return true if the specified CPU has any callback. If all_lazy is
+ * non-NULL, store an indication of whether all callbacks are lazy.
+ * (If there are no callbacks, all of them are deemed to be lazy.)
*/
-static int rcu_cpu_has_callbacks(int cpu)
+static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
{
+ bool al = true;
+ bool hc = false;
+ struct rcu_data *rdp;
struct rcu_state *rsp;
- /* RCU callbacks either ready or pending? */
- for_each_rcu_flavor(rsp)
- if (per_cpu_ptr(rsp-rda, cpu)-nxtlist)
- return 1;
- return 0;
+ for_each_rcu_flavor(rsp) {
+ rdp = per_cpu_ptr(rsp-rda, cpu);
+ if (rdp-qlen != rdp-qlen_lazy)
+ al = false;
+ if (rdp-nxtlist)
+ hc = true;
+ }
+ if (all_lazy)
+ *all_lazy = al;
+ return hc;
}
/*
@@ -2912,7 +2920,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int
preemptible)
rdp-dynticks-dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
atomic_set(rdp-dynticks-dynticks,
(atomic_read(rdp-dynticks-dynticks) ~0x1) + 1);
- rcu_prepare_for_idle_init(cpu);
raw_spin_unlock(rnp-lock);/* irqs remain disabled. */
/* Add CPU to rcu_node bitmasks. */
@@ -2986,7 +2993,6 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block
*self,
*/
for_each_rcu_flavor(rsp)
rcu_cleanup_dying_cpu(rsp);
- rcu_cleanup_after_idle(cpu);
break;
case CPU_DEAD:
case CPU_DEAD_FROZEN:
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index c865117..282b1d7 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -88,18 +88,13 @@ struct rcu_dynticks {
int dynticks_nmi_nesting; /* Track NMI nesting level. */
atomic_t dynticks; /* Even value for idle, else odd. */
#ifdef CONFIG_RCU_FAST_NO_HZ
- int dyntick_drain; /* Prepare-for-idle state variable. */
- unsigned long dyntick_holdoff;
- /* No retries for the jiffy of failure. */
- struct timer_list idle_gp_timer;
- /* Wake up CPU sleeping with callbacks. */
- unsigned long idle_gp_timer_expires;
- /* When to wake up CPU (for repost). */
- bool idle_first_pass; /* First pass of attempt to go idle? */
+ bool all_lazy; /* Are all CPU's CBs lazy? */
unsigned long nonlazy_posted;
/* # times non-lazy CBs posted to CPU. */
unsigned long nonlazy_posted_snap;
/* idle-period nonlazy_posted snapshot. */
+ unsigned long last_accelerate;
+ /* Last jiffy CBs were accelerated. */
int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
#ifdef CONFIG_RCU_USER_QS
@@ -530,7 +525,6 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct
rcu_state *rsp,
struct rcu_node *rnp);
#endif /* #ifdef
