On Fri, 2018-01-05 at 09:28 -0800, Linus Torvalds wrote: > That said, I honestly like the inline version (the one that is in the > google paper first) of the retpoline more than the out-of-line one. > And that one shouldn't have any relocagtion issues, because all the > offsets are relative. > > We want to use that one for the entry stub anyway, can't we just > standardize on that one for all our assembly? > > If the *compiler* uses the out-of-line version, that's a separate > thing. But for our asm cases, let's just make it all be the inline > case, ok?
OK, this one looks saner, and I think I've tested all the 32/64 bit retpoline/amd/none permutations. Pushed to http://git.infradead.org/users/dwmw2/linux-retpoline.git/ If this matches what you were thinking, I'll refactor the series in the morning to do it this way. From cfddb3bcae1524da52e782398da2809ec8faa200 Mon Sep 17 00:00:00 2001 From: David Woodhouse <d...@amazon.co.uk> Date: Fri, 5 Jan 2018 21:50:41 +0000 Subject: [PATCH] x86/retpoline: Clean up inverted X86_FEATURE_NO_RETPOLINE logic If we make the thunks inline so they don't need relocations to branch to __x86.indirect_thunk.xxx then we don't fall foul of the alternatives handling, and we can have the retpoline variant in 'altinstr'. This means that we can use X86_FEATURE_RETPOLINE which is more natural. Unfortunately, it does mean that the X86_FEATURE_K8 trick doesn't work any more, so we need an additional X86_FEATURE_RETPOLINE_AMD for that. Signed-off-by: David Woodhouse <d...@amazon.co.uk> --- arch/x86/entry/entry_64.S | 12 +-------- arch/x86/include/asm/cpufeatures.h | 3 ++- arch/x86/include/asm/nospec-branch.h | 50 +++++++++++++++++++++++++++--------- arch/x86/kernel/cpu/common.c | 5 ++++ arch/x86/kernel/cpu/intel.c | 3 ++- arch/x86/lib/retpoline.S | 17 +++++------- 6 files changed, 54 insertions(+), 36 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 76f94bbacaec..8f7e1129f493 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -188,17 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline) */ pushq %rdi movq $entry_SYSCALL_64_stage2, %rdi - /* - * Open-code the retpoline from retpoline.S, because we can't - * just jump to it directly. - */ - ALTERNATIVE "call 2f", "jmp *%rdi", X86_FEATURE_NO_RETPOLINE -1: - lfence - jmp 1b -2: - mov %rdi, (%rsp) - ret + NOSPEC_JMP rdi END(entry_SYSCALL_64_trampoline) .popsection diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 2d916fd13bf9..6f10edabbf82 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -203,7 +203,8 @@ #define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ #define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ -#define X86_FEATURE_NO_RETPOLINE ( 7*32+12) /* Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE ( 7*32+12) /* Intel Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ #define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */ #define X86_FEATURE_AVX512_4VNNIW ( 7*32+16) /* AVX-512 Neural Network Instructions */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index eced0dfaddc9..6e92edf64e53 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -8,23 +8,44 @@ #include <asm/cpufeatures.h> #ifdef __ASSEMBLY__ + /* - * The asm code uses CONFIG_RETPOLINE; this part will happen even if - * the toolchain isn't retpoline-capable. + * These are the bare retpoline primitives for indirect jmp and call. + * Do not use these directly. */ +.macro RETPOLINE_JMP reg:req + call 1112f +1111: lfence + jmp 1111b +1112: mov %\reg, (%_ASM_SP) + ret +.endm + +.macro RETPOLINE_CALL reg:req + jmp 1113f +1110: RETPOLINE_JMP \reg +1113: call 1110b +.endm + + + .macro NOSPEC_JMP reg:req #ifdef CONFIG_RETPOLINE - ALTERNATIVE __stringify(jmp __x86.indirect_thunk.\reg), __stringify(jmp *%\reg), X86_FEATURE_NO_RETPOLINE + ALTERNATIVE_2 __stringify(jmp *%\reg), \ + __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ + __stringify(lfence; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD #else - jmp *%\reg + jmp *%\reg #endif .endm .macro NOSPEC_CALL reg:req #ifdef CONFIG_RETPOLINE - ALTERNATIVE __stringify(call __x86.indirect_thunk.\reg), __stringify(call *%\reg), X86_FEATURE_NO_RETPOLINE + ALTERNATIVE_2 __stringify(call *%\reg), \ + __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ + __stringify(lfence; call *%\reg), X86_FEATURE_RETPOLINE_AMD #else - call *%\reg + call *%\reg #endif .endm @@ -36,16 +57,21 @@ */ #if defined(CONFIG_X86_64) && defined(RETPOLINE) # define NOSPEC_CALL ALTERNATIVE( \ + "call *%[thunk_target]\n", \ "call __x86.indirect_thunk.%V[thunk_target]\n", \ - "call *%[thunk_target]\n", X86_FEATURE_NO_RETPOLINE) + X86_FEATURE_RETPOLINE) # define THUNK_TARGET(addr) [thunk_target] "r" (addr) #elif defined(CONFIG_X86_64) && defined(CONFIG_RETPOLINE) # define NOSPEC_CALL ALTERNATIVE( \ - " jmp 1221f; " \ - "1222: push %[thunk_target];" \ - " jmp __x86.indirect_thunk;" \ - "1221: call 1222b;\n", \ - "call *%[thunk_target]\n", X86_FEATURE_NO_RETPOLINE) + "call *%[thunk_target]\n", \ + " jmp 1113f; " \ + "1110: call 1112f; " \ + "1111: lfence; " \ + " jmp 1111b; " \ + "1112: movl %[thunk_target], (%esp); " \ + " ret; " \ + "1113: call 1110b;\n", \ + X86_FEATURE_RETPOLINE) # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #else # define NOSPEC_CALL "call *%[thunk_target]\n" diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 372ba3fb400f..40e6e54d8501 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -904,6 +904,11 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c) setup_force_cpu_bug(X86_BUG_SPECTRE_V1); setup_force_cpu_bug(X86_BUG_SPECTRE_V2); +#ifdef CONFIG_RETPOLINE + setup_force_cpu_cap(X86_FEATURE_RETPOLINE); + if (c->x86_vendor == X86_VENDOR_AMD) + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); +#endif fpu__init_system(c); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index e1812d07b53e..35e123e5f413 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -35,7 +35,8 @@ static int __init noretpoline_setup(char *__unused) { pr_info("Retpoline runtime disabled\n"); - setup_force_cpu_cap(X86_FEATURE_NO_RETPOLINE); + setup_clear_cpu_cap(X86_FEATURE_RETPOLINE); + setup_clear_cpu_cap(X86_FEATURE_RETPOLINE_AMD); return 1; } __setup("noretpoline", noretpoline_setup); diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 2a4b1f09eb84..90d9a1589a54 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -6,19 +6,14 @@ #include <asm/cpufeatures.h> #include <asm/alternative-asm.h> #include <asm/export.h> +#include <asm/nospec-branch.h> -.macro THUNK sp reg +.macro THUNK reg .section .text.__x86.indirect_thunk.\reg ENTRY(__x86.indirect_thunk.\reg) CFI_STARTPROC - ALTERNATIVE_2 "call 2f", __stringify(lfence;jmp *%\reg), X86_FEATURE_K8, __stringify(jmp *%\reg), X86_FEATURE_NO_RETPOLINE -1: - lfence - jmp 1b -2: - mov %\reg, (%\sp) - ret + NOSPEC_JMP \reg CFI_ENDPROC ENDPROC(__x86.indirect_thunk.\reg) EXPORT_SYMBOL(__x86.indirect_thunk.\reg) @@ -26,11 +21,11 @@ EXPORT_SYMBOL(__x86.indirect_thunk.\reg) #ifdef CONFIG_64BIT .irp reg rax rbx rcx rdx rsi rdi rbp r8 r9 r10 r11 r12 r13 r14 r15 - THUNK rsp \reg + THUNK \reg .endr #else .irp reg eax ebx ecx edx esi edi ebp - THUNK esp \reg + THUNK \reg .endr /* @@ -39,7 +34,7 @@ EXPORT_SYMBOL(__x86.indirect_thunk.\reg) */ ENTRY(__x86.indirect_thunk) CFI_STARTPROC - ALTERNATIVE "call 2f", "ret", X86_FEATURE_NO_RETPOLINE + ALTERNATIVE "ret", "call 2f", X86_FEATURE_RETPOLINE 1: lfence jmp 1b -- 2.14.3
smime.p7s
Description: S/MIME cryptographic signature
