From: Borislav Petkov <[email protected]> Simplify it to call an asm-function instead of pasting 41 insn bytes at every call site. Also, add alignment to the macro as suggested here:
https://support.google.com/faqs/answer/7625886 Signed-off-by: Borislav Petkov <[email protected]> Cc: David Woodhouse <[email protected]> --- arch/x86/include/asm/nospec-branch.h | 39 +++++++++++++++++++++++++++--------- arch/x86/include/asm/processor.h | 5 +++++ arch/x86/lib/Makefile | 1 + arch/x86/lib/retpoline.S | 5 +++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 33a35daf6c4d..61d4d7033758 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -53,6 +53,8 @@ #ifdef __ASSEMBLY__ +#include <asm/bitsperlong.h> + /* * This should be used immediately before a retpoline alternative. It tells * objtool where the retpolines are so that it can make sense of the control @@ -121,6 +123,30 @@ #endif .endm +/* Same as above but with alignment additionally */ +.macro ___FILL_RETURN_BUFFER reg:req nr:req sp:req + mov (\nr / 2), \reg + .align 16 +771: + call 772f +773: /* speculation trap */ + pause + lfence + jmp 773b + .align 16 +772: + call 774f +775: /* speculation trap */ + pause + lfence + jmp 775b + .align 16 +774: + dec \reg + jnz 771b + add (BITS_PER_LONG/8) * \nr, \sp +.endm + /* * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP * monstrosity above, manually. @@ -206,15 +232,10 @@ extern char __indirect_thunk_end[]; static inline void vmexit_fill_RSB(void) { #ifdef CONFIG_RETPOLINE - unsigned long loops; - - asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE - ALTERNATIVE("jmp 910f", - __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), - X86_FEATURE_RETPOLINE) - "910:" - : "=r" (loops), ASM_CALL_CONSTRAINT - : : "memory" ); + alternative_input("", + "call __fill_rsb_clobber_ax", + X86_FEATURE_RETPOLINE, + ASM_NO_INPUT_CLOBBER(_ASM_AX, "memory")); #endif } diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index d3a67fba200a..491f6e0be66e 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -971,4 +971,9 @@ bool xen_set_default_idle(void); void stop_this_cpu(void *dummy); void df_debug(struct pt_regs *regs, long error_code); + +#ifdef CONFIG_RETPOLINE +asmlinkage void __fill_rsb_clobber_ax(void); +#endif + #endif /* _ASM_X86_PROCESSOR_H */ diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f23934bbaf4e..69a473919260 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -27,6 +27,7 @@ lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o lib-$(CONFIG_RETPOLINE) += retpoline.o +OBJECT_FILES_NON_STANDARD_retpoline.o :=y obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index c909961e678a..297b0fd2ad10 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -46,3 +46,8 @@ GENERATE_THUNK(r13) GENERATE_THUNK(r14) GENERATE_THUNK(r15) #endif + +ENTRY(__fill_rsb_clobber_ax) + ___FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, %_ASM_SP +END(__fill_rsb_clobber_ax) +EXPORT_SYMBOL_GPL(__fill_rsb_clobber_ax) -- 2.13.0
