Currently, on arm64 platforms, the handle_arch_irq is a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5% Signed-off-by: Jisheng Zhang <[email protected]> --- arch/arm64/kernel/entry-common.c | 4 +++- arch/arm64/kernel/irq.c | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 3625797e9ee8..46a4c012e15f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -25,6 +25,7 @@ #include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/processor.h> +#include <asm/runtime-const.h> #include <asm/sdei.h> #include <asm/stacktrace.h> #include <asm/sysreg.h> @@ -139,7 +140,8 @@ static void do_interrupt_handler(struct pt_regs *regs, set_irq_regs(old_regs); } -extern void (*handle_arch_irq)(struct pt_regs *); +extern void (*_handle_arch_irq)(struct pt_regs *); +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq) extern void (*handle_arch_fiq)(struct pt_regs *); static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 15dedb385b9e..30629c183606 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -23,6 +23,7 @@ #include <asm/daifflags.h> #include <asm/exception.h> #include <asm/numa.h> +#include <asm/runtime-const.h> #include <asm/softirq_stack.h> #include <asm/stacktrace.h> #include <asm/vmap_stack.h> @@ -84,15 +85,17 @@ static void default_handle_fiq(struct pt_regs *regs) panic("FIQ taken without a root FIQ handler\n"); } -void (*handle_arch_irq)(struct pt_regs *) __ro_after_init = default_handle_irq; +void (*_handle_arch_irq)(struct pt_regs *) __ro_after_init = default_handle_irq; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq) void (*handle_arch_fiq)(struct pt_regs *) __ro_after_init = default_handle_fiq; int __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) { - if (handle_arch_irq != default_handle_irq) + if (_handle_arch_irq != default_handle_irq) return -EBUSY; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; + runtime_const_init(ptr, _handle_arch_irq); pr_info("Root IRQ handler: %ps\n", handle_irq); return 0; } -- 2.51.0
