__do_irq() inconditionnaly calls ppc_md.get_irq()
That's definitely a hot path.
At the time being ppc_md.get_irq address is read every time
from ppc_md structure.
Replace that call by a static call, and initialise that
call after ppc_md.init_IRQ() has set ppc_md.get_irq.
Emit a warning and don't set the static call if ppc_md.init_IRQ()
is still NULL, that way the kernel won't blow up if for some
reason ppc_md.get_irq() doesn't get properly set.
With the patch:
00000000 <__SCT__ppc_get_irq>:
0: 48 00 00 20 b 20 <__static_call_return0> <==
Replaced by 'b <ppc_md.get_irq>' at runtime
...
00000020 <__static_call_return0>:
20: 38 60 00 00 li r3,0
24: 4e 80 00 20 blr
...
00000058 <__do_irq>:
...
64: 48 00 00 01 bl 64 <__do_irq+0xc>
64: R_PPC_REL24 __SCT__ppc_get_irq
68: 2c 03 00 00 cmpwi r3,0
...
Before the patch:
00000038 <__do_irq>:
...
3c: 3d 20 00 00 lis r9,0
3e: R_PPC_ADDR16_HA ppc_md+0x1c
...
44: 81 29 00 00 lwz r9,0(r9)
46: R_PPC_ADDR16_LO ppc_md+0x1c
...
4c: 7d 29 03 a6 mtctr r9
50: 4e 80 04 21 bctrl
54: 2c 03 00 00 cmpwi r3,0
...
On PPC64 which doesn't implement static calls yet we get:
00000000000000d0 <__do_irq>:
...
dc: 00 00 22 3d addis r9,r2,0
dc: R_PPC64_TOC16_HA .data+0x8
...
e4: 00 00 89 e9 ld r12,0(r9)
e4: R_PPC64_TOC16_LO_DS .data+0x8
...
f0: a6 03 89 7d mtctr r12
f4: 18 00 41 f8 std r2,24(r1)
f8: 21 04 80 4e bctrl
fc: 18 00 41 e8 ld r2,24(r1)
...
So on PPC64 that's similar to what we get without static calls.
But at least until ppc_md.get_irq() is set the call is to
__static_call_return0.
Signed-off-by: Christophe Leroy <christophe.le...@csgroup.eu>
---
arch/powerpc/kernel/irq.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 752fb182eacb..1c4715a03cd1 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -52,6 +52,7 @@
#include <linux/of_irq.h>
#include <linux/vmalloc.h>
#include <linux/pgtable.h>
+#include <linux/static_call.h>
#include <linux/uaccess.h>
#include <asm/interrupt.h>
@@ -730,6 +731,8 @@ static __always_inline void call_do_irq(struct pt_regs
*regs, void *sp)
);
}
+DEFINE_STATIC_CALL_RET0(ppc_get_irq, *ppc_md.get_irq);
+
void __do_irq(struct pt_regs *regs)
{
unsigned int irq;
@@ -741,7 +744,7 @@ void __do_irq(struct pt_regs *regs)
*
* This will typically lower the interrupt line to the CPU
*/
- irq = ppc_md.get_irq();
+ irq = static_call(ppc_get_irq)();
/* We can hard enable interrupts now to allow perf interrupts */
if (should_hard_irq_enable())
@@ -809,6 +812,9 @@ void __init init_IRQ(void)
if (ppc_md.init_IRQ)
ppc_md.init_IRQ();
+
+ if (!WARN_ON(!ppc_md.get_irq))
+ static_call_update(ppc_get_irq, ppc_md.get_irq);
}
#ifdef CONFIG_BOOKE_OR_40x
--
2.34.1
