Now that all callers are ready, we can push down the softirq enabled mask to the core from callers such as spin_lock_bh(), local_bh_disable(), rcu_read_lock_bh(), etc...
It is applied to the CPU vector enabled mask on __local_bh_disable_ip() which then returns the old value to be restored on __local_bh_enable_ip(). Signed-off-by: Frederic Weisbecker <frede...@kernel.org> Cc: Ingo Molnar <mi...@kernel.org> Cc: Sebastian Andrzej Siewior <bige...@linutronix.de> Cc: Thomas Gleixner <t...@linutronix.de> Cc: Peter Zijlstra <pet...@infradead.org> Cc: Linus Torvalds <torva...@linux-foundation.org> Cc: David S. Miller <da...@davemloft.net> Cc: Mauro Carvalho Chehab <mche...@s-opensource.com> Cc: Paul E. McKenney <paul...@linux.vnet.ibm.com> --- include/linux/bottom_half.h | 19 ++++++++++--------- include/linux/rwlock_api_smp.h | 14 ++++++++------ include/linux/spinlock_api_smp.h | 10 +++++----- kernel/softirq.c | 28 +++++++++++++++++++--------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/include/linux/bottom_half.h b/include/linux/bottom_half.h index 31fcdae..f8a68c8 100644 --- a/include/linux/bottom_half.h +++ b/include/linux/bottom_half.h @@ -37,9 +37,10 @@ enum #ifdef CONFIG_TRACE_IRQFLAGS -extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); +extern unsigned int __local_bh_disable_ip(unsigned long ip, unsigned int cnt, + unsigned int mask); #else -static __always_inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) +static __always_inline unsigned int __local_bh_disable_ip(unsigned long ip, unsigned int cnt) { preempt_count_add(cnt); barrier(); @@ -48,21 +49,21 @@ static __always_inline void __local_bh_disable_ip(unsigned long ip, unsigned int static inline unsigned int local_bh_disable(unsigned int mask) { - __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET); - return 0; + return __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET, mask); } -extern void local_bh_enable_no_softirq(void); -extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt); +extern void local_bh_enable_no_softirq(unsigned int bh); +extern void __local_bh_enable_ip(unsigned long ip, + unsigned int cnt, unsigned int bh); -static inline void local_bh_enable_ip(unsigned long ip) +static inline void local_bh_enable_ip(unsigned long ip, unsigned int bh) { - __local_bh_enable_ip(ip, SOFTIRQ_DISABLE_OFFSET); + __local_bh_enable_ip(ip, SOFTIRQ_DISABLE_OFFSET, bh); } static inline void local_bh_enable(unsigned int bh) { - __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET); + __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET, bh); } extern void local_bh_disable_all(void); diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index fb66489..90ba7bf 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -173,10 +173,11 @@ static inline void __raw_read_lock_irq(rwlock_t *lock) static inline unsigned int __raw_read_lock_bh(rwlock_t *lock, unsigned int mask) { - __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + unsigned int bh; + bh = __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, mask); rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock); - return 0; + return bh; } static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock) @@ -202,10 +203,11 @@ static inline void __raw_write_lock_irq(rwlock_t *lock) static inline unsigned int __raw_write_lock_bh(rwlock_t *lock, unsigned int mask) { - __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + unsigned int bh; + bh = __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, mask); rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock); - return 0; + return bh; } static inline void __raw_write_lock(rwlock_t *lock) @@ -253,7 +255,7 @@ static inline void __raw_read_unlock_bh(rwlock_t *lock, { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_read_unlock(lock); - __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, bh); } static inline void __raw_write_unlock_irqrestore(rwlock_t *lock, @@ -278,7 +280,7 @@ static inline void __raw_write_unlock_bh(rwlock_t *lock, { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_write_unlock(lock); - __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, bh); } #endif /* __LINUX_RWLOCK_API_SMP_H */ diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 42bbf68..6602a56 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -132,9 +132,9 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) static inline unsigned int __raw_spin_lock_bh(raw_spinlock_t *lock, unsigned int mask) { - unsigned int bh = 0; + unsigned int bh; - __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + bh = __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, mask); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); @@ -179,19 +179,19 @@ static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock, { spin_release(&lock->dep_map, 1, _RET_IP_); do_raw_spin_unlock(lock); - __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, bh); } static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock, unsigned int *bh, unsigned int mask) { - __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + *bh = __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, mask); if (do_raw_spin_trylock(lock)) { spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); return 1; } - __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); + __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET, *bh); return 0; } diff --git a/kernel/softirq.c b/kernel/softirq.c index 22cc0a7..e2435b0 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -107,13 +107,16 @@ static bool ksoftirqd_running(unsigned long pending) * where hardirqs are disabled legitimately: */ #ifdef CONFIG_TRACE_IRQFLAGS -void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) +unsigned int __local_bh_disable_ip(unsigned long ip, unsigned int cnt, + unsigned int mask) { unsigned long flags; + unsigned int enabled; WARN_ON_ONCE(in_irq()); raw_local_irq_save(flags); + /* * The preempt tracer hooks into preempt_count_add and will break * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET @@ -127,6 +130,9 @@ void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) */ if (softirq_count() == (cnt & SOFTIRQ_MASK)) trace_softirqs_off(ip); + + enabled = local_softirq_enabled(); + softirq_enabled_nand(mask); raw_local_irq_restore(flags); if (preempt_count() == cnt) { @@ -135,6 +141,7 @@ void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) #endif trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip()); } + return enabled; } EXPORT_SYMBOL(__local_bh_disable_ip); #endif /* CONFIG_TRACE_IRQFLAGS */ @@ -143,11 +150,13 @@ EXPORT_SYMBOL(__local_bh_disable_ip); * Special-case - softirqs can safely be enabled by __do_softirq(), * without processing still-pending softirqs: */ -void local_bh_enable_no_softirq(void) +void local_bh_enable_no_softirq(unsigned int bh) { WARN_ON_ONCE(in_irq()); lockdep_assert_irqs_disabled(); + softirq_enabled_set(bh); + if (preempt_count() == SOFTIRQ_DISABLE_OFFSET) trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); @@ -155,17 +164,18 @@ void local_bh_enable_no_softirq(void) trace_softirqs_on(_RET_IP_); __preempt_count_sub(SOFTIRQ_DISABLE_OFFSET); - } EXPORT_SYMBOL(local_bh_enable_no_softirq); -void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) +void __local_bh_enable_ip(unsigned long ip, unsigned int cnt, unsigned int bh) { WARN_ON_ONCE(in_irq()); lockdep_assert_irqs_enabled(); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_disable(); #endif + softirq_enabled_set(bh); + /* * Are softirqs going to be turned on now: */ @@ -177,6 +187,7 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) */ preempt_count_sub(cnt - 1); + if (unlikely(!in_interrupt() && local_softirq_pending())) { /* * Run softirq if any pending. And do it in its own stack @@ -246,9 +257,6 @@ static void local_bh_exit(void) __preempt_count_sub(SOFTIRQ_OFFSET); } - - - /* * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times, * but break the loop if need_resched() is set or after 2 ms. @@ -395,15 +403,17 @@ asmlinkage __visible void do_softirq(void) */ void irq_enter(void) { + unsigned int bh; + rcu_irq_enter(); if (is_idle_task(current) && !in_interrupt()) { /* * Prevent raise_softirq from needlessly waking up ksoftirqd * here, as softirq will be serviced on return from interrupt. */ - local_bh_disable(SOFTIRQ_ALL_MASK); + bh = local_bh_disable(SOFTIRQ_ALL_MASK); tick_irq_enter(); - local_bh_enable_no_softirq(); + local_bh_enable_no_softirq(bh); } __irq_enter(); -- 2.7.4