extable is used for handling user memory access faults from kernel mode,
as such it is a fast-ish path. Using it in the very slow WARN path
increases the number of extable entries from 1306 to 6516 in a
ppc64le_defconfig vmlinux, increasing the average number of search
steps by ~2.3.
This patch adds a recovery address to the bug_entry struct instead of
using the extable. The size of the bug entry table plus the extable
go from 137kB to 126kB.
Signed-off-by: Nicholas Piggin <npig...@gmail.com>
---
arch/powerpc/Kconfig | 4 ++++
arch/powerpc/include/asm/bug.h | 31 ++++++++++++++++++++++++++++---
arch/powerpc/kernel/traps.c | 16 ++++++++++------
include/asm-generic/bug.h | 3 +++
4 files changed, 45 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 4c466acdc70d..3b379fa15082 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -336,6 +336,10 @@ config GENERIC_BUG_RELATIVE_POINTERS
def_bool y
depends on GENERIC_BUG
+config GENERIC_BUG_ARCH_ENTRY
+ def_bool y
+ depends on GENERIC_BUG
+
config SYS_SUPPORTS_APM_EMULATION
default y if PMAC_APM_EMU
bool
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index 61a4736355c2..dec73137fea0 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -14,6 +14,7 @@
.macro __EMIT_BUG_ENTRY addr,file,line,flags
.section __bug_table,"aw"
5001: .4byte \addr - .
+ .4byte 0
.4byte 5002f - .
.short \line, \flags
.org 5001b+BUG_ENTRY_SIZE
@@ -26,6 +27,7 @@
.macro __EMIT_BUG_ENTRY addr,file,line,flags
.section __bug_table,"aw"
5001: .4byte \addr - .
+ .4byte 0
.short \flags
.org 5001b+BUG_ENTRY_SIZE
.previous
@@ -33,7 +35,6 @@
#endif /* verbose */
.macro EMIT_WARN_ENTRY addr,file,line,flags
- EX_TABLE(\addr,\addr+4)
__EMIT_BUG_ENTRY \addr,\file,\line,\flags
.endm
@@ -45,12 +46,17 @@
.endm
#else /* !__ASSEMBLY__ */
+struct arch_bug_entry {
+ signed int recovery_addr_disp;
+};
+
/* _EMIT_BUG_ENTRY expects args %0,%1,%2,%3 to be FILE, LINE, flags and
sizeof(struct bug_entry), respectively */
#ifdef CONFIG_DEBUG_BUGVERBOSE
#define _EMIT_BUG_ENTRY \
".section __bug_table,\"aw\"\n" \
"2: .4byte 1b - .\n" \
+ " .4byte 0\n" \
" .4byte %0 - .\n" \
" .short %1, %2\n" \
".org 2b+%3\n" \
@@ -59,6 +65,26 @@
#define _EMIT_BUG_ENTRY \
".section __bug_table,\"aw\"\n" \
"2: .4byte 1b - .\n" \
+ " .4byte 0\n" \
+ " .short %2\n" \
+ ".org 2b+%3\n" \
+ ".previous\n"
+#endif
+
+#ifdef CONFIG_DEBUG_BUGVERBOSE
+#define _EMIT_WARN_ENTRY(label) \
+ ".section __bug_table,\"aw\"\n" \
+ "2: .4byte 1b - .\n" \
+ " .4byte 1b - %l[" #label "]\n" \
+ " .4byte %0 - .\n" \
+ " .short %1, %2\n" \
+ ".org 2b+%3\n" \
+ ".previous\n"
+#else
+#define _EMIT_WARN_ENTRY(label) \
+ ".section __bug_table,\"aw\"\n" \
+ "2: .4byte 1b - .\n" \
+ " .4byte 1b - %l[" #label "]\n" \
" .short %2\n" \
".org 2b+%3\n" \
".previous\n"
@@ -76,8 +102,7 @@
#define WARN_ENTRY(insn, flags, label, ...) \
asm_volatile_goto( \
"1: " insn "\n" \
- EX_TABLE(1b, %l[label]) \
- _EMIT_BUG_ENTRY \
+ _EMIT_WARN_ENTRY(label) \
: : "i" (__FILE__), "i" (__LINE__), \
"i" (flags), \
"i" (sizeof(struct bug_entry)), \
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index dadfcef5d6db..1e521a432bd0 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1494,13 +1494,17 @@ static void do_program_check(struct pt_regs *regs)
if (!(regs->msr & MSR_PR) && /* not user-mode */
report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) {
- const struct exception_table_entry *entry;
+ const struct bug_entry *bug;
+ unsigned long recov;
- entry = search_exception_tables(bugaddr);
- if (entry) {
- regs_set_return_ip(regs, extable_fixup(entry) +
regs->nip - bugaddr);
- return;
- }
+ bug = find_bug(bugaddr);
+ if (!bug || bug->arch.recovery_addr_disp == 0)
+ recov = regs->nip + 4;
+ else
+ recov = bugaddr - bug->arch.recovery_addr_disp;
+
+ regs_set_return_ip(regs, recov);
+ return;
}
_exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
return;
diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h
index ba1f860af38b..1ce8431bdf02 100644
--- a/include/asm-generic/bug.h
+++ b/include/asm-generic/bug.h
@@ -36,6 +36,9 @@ struct bug_entry {
#else
signed int bug_addr_disp;
#endif
+#ifdef CONFIG_GENERIC_BUG_ARCH_ENTRY
+ struct arch_bug_entry arch;
+#endif
#ifdef CONFIG_DEBUG_BUGVERBOSE
#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
const char *file;
--
2.37.2
