As we plan to be able to defer some specific softirq vector processing
to workqueues when those vectors need more time than IRQs can offer,
let's first introduce the per-vector call counter/limit.
Each softirq vector is allowed to be called on IRQ tail at most
MAX_SOFTIRQ_RESTART per jiffy. Once we reach that limit, the softirq
processing is deferred to ksoftirqd. The threading will be divided to
per vector worklets in further patches.
Suggested-by: Linus Torvalds <torva...@linux-foundation.org>
Signed-off-by: Frederic Weisbecker <frede...@kernel.org>
Cc: Dmitry Safonov <d...@arista.com>
Cc: Eric Dumazet <eduma...@google.com>
Cc: Linus Torvalds <torva...@linux-foundation.org>
Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Andrew Morton <a...@linux-foundation.org>
Cc: David Miller <da...@davemloft.net>
Cc: Hannes Frederic Sowa <han...@stressinduktion.org>
Cc: Ingo Molnar <mi...@kernel.org>
Cc: Levin Alexander <alexander.le...@verizon.com>
Cc: Paolo Abeni <pab...@redhat.com>
Cc: Paul E. McKenney <paul...@linux.vnet.ibm.com>
Cc: Radu Rendec <rren...@arista.com>
Cc: Rik van Riel <r...@redhat.com>
Cc: Stanislaw Gruszka <sgrus...@redhat.com>
Cc: Thomas Gleixner <t...@linutronix.de>
Cc: Wanpeng Li <wanpeng...@hotmail.com>
Cc: Mauro Carvalho Chehab <mche...@s-opensource.com>
---
kernel/softirq.c | 41 ++++++++++++++++++++++++++---------------
1 file changed, 26 insertions(+), 15 deletions(-)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 2f5e87f..e0f4b29 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -62,6 +62,15 @@ const char * const softirq_to_name[NR_SOFTIRQS] = {
"TASKLET", "SCHED", "HRTIMER", "RCU"
};
+struct vector {
+ unsigned int jiffy_calls;
+ unsigned long jiffy_snap;
+};
+
+static DEFINE_PER_CPU(struct vector, vector_cpu[NR_SOFTIRQS]) = {
+ [0 ... NR_SOFTIRQS-1] = { 0, INITIAL_JIFFIES }
+};
+
/*
* we cannot loop indefinitely here to avoid userspace starvation,
* but we also don't want to introduce a worst case 1/HZ latency
@@ -192,19 +201,13 @@ EXPORT_SYMBOL(__local_bh_enable_ip);
/*
* We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
- * but break the loop if need_resched() is set or after 2 ms.
- * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
- * certain cases, such as stop_machine(), jiffies may cease to
- * increment and so we need the MAX_SOFTIRQ_RESTART limit as
- * well to make sure we eventually return from this method.
- *
+ * but break the loop if need_resched() is set.
* These limits have been established via experimentation.
* The two things to balance is latency against fairness -
* we want to handle softirqs as soon as possible, but they
* should not be able to lock up the box.
*/
-#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
-#define MAX_SOFTIRQ_RESTART 10
+#define MAX_SOFTIRQ_RESTART 20
#ifdef CONFIG_TRACE_IRQFLAGS
/*
@@ -241,12 +244,10 @@ static inline void lockdep_softirq_end(bool in_hardirq) {
}
asmlinkage __visible void __softirq_entry __do_softirq(void)
{
- unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
unsigned long old_flags = current->flags;
- int max_restart = MAX_SOFTIRQ_RESTART;
struct softirq_action *h;
bool in_hardirq;
- __u32 pending;
+ __u32 pending, overrun = 0;
int softirq_bit;
/*
@@ -271,6 +272,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void)
h = softirq_vec;
while ((softirq_bit = ffs(pending))) {
+ struct vector *vector;
unsigned int vec_nr;
int prev_count;
@@ -284,6 +286,16 @@ asmlinkage __visible void __softirq_entry
__do_softirq(void)
trace_softirq_entry(vec_nr);
h->action(h);
trace_softirq_exit(vec_nr);
+
+ vector = this_cpu_ptr(&vector_cpu[vec_nr]);
+ if (time_before(vector->jiffy_snap, jiffies)) {
+ vector->jiffy_calls = 0;
+ vector->jiffy_snap = jiffies;
+ }
+
+ if (++vector->jiffy_calls > MAX_SOFTIRQ_RESTART)
+ overrun |= 1 << vec_nr;
+
if (unlikely(prev_count != preempt_count())) {
pr_err("huh, entered softirq %u %s %p with
preempt_count %08x, exited with %08x?\n",
vec_nr, softirq_to_name[vec_nr], h->action,
@@ -299,11 +311,10 @@ asmlinkage __visible void __softirq_entry
__do_softirq(void)
pending = local_softirq_pending();
if (pending) {
- if (time_before(jiffies, end) && !need_resched() &&
- --max_restart)
+ if (overrun || need_resched())
+ wakeup_softirqd();
+ else
goto restart;
-
- wakeup_softirqd();
}
lockdep_softirq_end(in_hardirq);
--
2.7.4
