In preparation of permitting callback type alternatives patching routines to use any number of alternative instructions, move the relative reference to the callback routine out of the primary 'struct alt_instr' data structure, where we are currently overloading the 'alt_offset' member depending on the feature id. Instead, put a relative reference to the callback at the start of the alternative sequence.
Signed-off-by: Ard Biesheuvel <[email protected]> --- arch/arm64/include/asm/alternative.h | 39 +++++++++----------- arch/arm64/kernel/alternative.c | 11 ++++-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index 4b650ec1d7dd..77da798e888b 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -24,6 +24,11 @@ struct alt_instr { u8 alt_len; /* size of new instruction(s), <= orig_len */ }; +struct alt_instr_cb { + s32 cb_offset; /* offset to callback handler */ + __le32 insn[]; /* sequence of alternative instructions */ +}; + typedef void (*alternative_cb_t)(struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst); @@ -35,13 +40,9 @@ void apply_alternatives_module(void *start, size_t length); static inline void apply_alternatives_module(void *start, size_t length) { } #endif -#define ALTINSTR_ENTRY(feature,cb) \ +#define ALTINSTR_ENTRY(feature) \ " .word 661b - .\n" /* label */ \ - " .if " __stringify(cb) " == 0\n" \ " .word 663f - .\n" /* new instruction */ \ - " .else\n" \ - " .word " __stringify(cb) "- .\n" /* callback */ \ - " .endif\n" \ " .hword " __stringify(feature) "\n" /* feature bit */ \ " .byte 662b-661b\n" /* source len */ \ " .byte 664f-663f\n" /* replacement len */ @@ -59,36 +60,29 @@ static inline void apply_alternatives_module(void *start, size_t length) { } * but most assemblers die if insn1 or insn2 have a .inst. This should * be fixed in a binutils release posterior to 2.25.51.0.2 (anything * containing commit 4e4d08cf7399b606 or c1baaddf8861). - * - * Alternatives with callbacks do not generate replacement instructions. */ -#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled, cb) \ - ".if "__stringify(cfg_enabled)" == 1\n" \ +#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature) \ "661:\n\t" \ oldinstr "\n" \ "662:\n" \ ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY(feature,cb) \ + ALTINSTR_ENTRY(feature) \ ".popsection\n" \ - " .if " __stringify(cb) " == 0\n" \ ".pushsection .altinstr_replacement, \"a\"\n" \ "663:\n\t" \ newinstr "\n" \ "664:\n\t" \ - ".popsection\n\t" \ + ".popsection\n\t" + +#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ + ".if "__stringify(IS_ENABLED(cfg))" == 1\n" \ + __ALTERNATIVE_CFG(oldinstr, newinstr, feature) \ ".org . - (664b-663b) + (662b-661b)\n\t" \ ".org . - (662b-661b) + (664b-663b)\n" \ - ".else\n\t" \ - "663:\n\t" \ - "664:\n\t" \ - ".endif\n" \ ".endif\n" -#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ - __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg), 0) - #define ALTERNATIVE_CB(oldinstr, cb) \ - __ALTERNATIVE_CFG(oldinstr, "NOT_AN_INSTRUCTION", ARM64_CB_PATCH, 1, cb) + __ALTERNATIVE_CFG(oldinstr, ".word " __stringify(cb) " - .\n", ARM64_CB_PATCH) #else #include <asm/assembler.h> @@ -158,8 +152,11 @@ static inline void apply_alternatives_module(void *start, size_t length) { } .macro alternative_cb cb .set .Lasm_alt_mode, 0 .pushsection .altinstructions, "a" - altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0 + altinstruction_entry 661f, 663f, ARM64_CB_PATCH, 662f-661f, 664f-663f .popsection + .pushsection .altinstr_replacement, "ax" +663: .word \cb - . +664: .popsection 661: .endm diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index b5d603992d40..a49930843784 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -151,6 +151,7 @@ static void __apply_alternatives(void *alt_region, bool is_module) struct alt_region *region = alt_region; __le32 *origptr, *updptr; alternative_cb_t alt_cb; + struct alt_instr_cb *alt_cb_insn; for (alt = region->begin; alt < region->end; alt++) { int nr_inst; @@ -161,7 +162,7 @@ static void __apply_alternatives(void *alt_region, bool is_module) continue; if (alt->cpufeature == ARM64_CB_PATCH) - BUG_ON(alt->alt_len != 0); + BUG_ON(alt->alt_len < sizeof(*alt_cb_insn)); else BUG_ON(alt->alt_len != alt->orig_len); @@ -171,10 +172,12 @@ static void __apply_alternatives(void *alt_region, bool is_module) updptr = is_module ? origptr : lm_alias(origptr); nr_inst = alt->orig_len / AARCH64_INSN_SIZE; - if (alt->cpufeature < ARM64_CB_PATCH) + if (alt->cpufeature < ARM64_CB_PATCH) { alt_cb = patch_alternative; - else - alt_cb = ALT_REPL_PTR(alt); + } else { + alt_cb_insn = ALT_REPL_PTR(alt); + alt_cb = offset_to_ptr(&alt_cb_insn->cb_offset); + } alt_cb(alt, origptr, updptr, nr_inst); -- 2.19.2
