[tip:x86/pti] x86/retpoline: Support retpoline builds with Clang

[tip-bot for David Woodhouse]

Commit-ID:  87358710c1fb4f1bf96bbe2349975ff9953fc9b2
Gitweb: https://git.kernel.org/tip/87358710c1fb4f1bf96bbe2349975ff9953fc9b2
Author: David Woodhouse 
AuthorDate: Mon, 19 Feb 2018 10:50:57 +
Committer:  Ingo Molnar 
CommitDate: Tue, 20 Feb 2018 11:17:58 +0100

x86/retpoline: Support retpoline builds with Clang

Signed-off-by: David Woodhouse 
Reviewed-by: Thomas Gleixner 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: arjan.van.de@intel.com
Cc: b...@alien8.de
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1519037457-7643-5-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/Makefile  | 5 -
 include/linux/compiler-clang.h | 5 +
 include/linux/compiler-gcc.h   | 4 
 include/linux/init.h   | 8 
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index fad5516..dbc7d0e 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -232,7 +232,10 @@ KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
 # Avoid indirect branches in kernel to deal with Spectre
 ifdef CONFIG_RETPOLINE
-RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern 
-mindirect-branch-register
+RETPOLINE_CFLAGS_CLANG := -mretpoline-external-thunk
+
+RETPOLINE_CFLAGS += $(call cc-option,$(RETPOLINE_CFLAGS_GCC),$(call 
cc-option,$(RETPOLINE_CFLAGS_CLANG)))
 ifneq ($(RETPOLINE_CFLAGS),)
 KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
 endif
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index d02a4df..d3f264a 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -27,3 +27,8 @@
 #if __has_feature(address_sanitizer)
 #define __SANITIZE_ADDRESS__
 #endif
+
+/* Clang doesn't have a way to turn it off per-function, yet. */
+#ifdef __noretpoline
+#undef __noretpoline
+#endif
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index 73bc63e..673fbf9 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -93,6 +93,10 @@
 #define __weak __attribute__((weak))
 #define __alias(symbol)__attribute__((alias(#symbol)))
 
+#ifdef RETPOLINE
+#define __noretpoline __attribute__((indirect_branch("keep")))
+#endif
+
 /*
  * it doesn't make sense on ARM (currently the only user of __naked)
  * to trace naked functions because then mcount is called without
diff --git a/include/linux/init.h b/include/linux/init.h
index 506a981..bc27cf0 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -6,10 +6,10 @@
 #include 
 
 /* Built-in __init functions needn't be compiled with retpoline */
-#if defined(RETPOLINE) && !defined(MODULE)
-#define __noretpoline __attribute__((indirect_branch("keep")))
+#if defined(__noretpoline) && !defined(MODULE)
+#define __noinitretpoline __noretpoline
 #else
-#define __noretpoline
+#define __noinitretpoline
 #endif
 
 /* These macros are used to mark some functions or 
@@ -47,7 +47,7 @@
 
 /* These are for everybody (although not all archs will actually
discard it in modules) */
-#define __init __section(.init.text) __cold  __latent_entropy 
__noretpoline
+#define __init __section(.init.text) __cold  __latent_entropy 
__noinitretpoline
 #define __initdata __section(.init.data)
 #define __initconst__section(.init.rodata)
 #define __exitdata __section(.exit.data)

[tip:x86/pti] x86/retpoline: Support retpoline builds with Clang

[tip-bot for David Woodhouse]

Commit-ID:  87358710c1fb4f1bf96bbe2349975ff9953fc9b2
Gitweb: https://git.kernel.org/tip/87358710c1fb4f1bf96bbe2349975ff9953fc9b2
Author: David Woodhouse 
AuthorDate: Mon, 19 Feb 2018 10:50:57 +
Committer:  Ingo Molnar 
CommitDate: Tue, 20 Feb 2018 11:17:58 +0100

x86/retpoline: Support retpoline builds with Clang

Signed-off-by: David Woodhouse 
Reviewed-by: Thomas Gleixner 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: arjan.van.de@intel.com
Cc: b...@alien8.de
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1519037457-7643-5-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/Makefile  | 5 -
 include/linux/compiler-clang.h | 5 +
 include/linux/compiler-gcc.h   | 4 
 include/linux/init.h   | 8 
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index fad5516..dbc7d0e 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -232,7 +232,10 @@ KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
 # Avoid indirect branches in kernel to deal with Spectre
 ifdef CONFIG_RETPOLINE
-RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern 
-mindirect-branch-register
+RETPOLINE_CFLAGS_CLANG := -mretpoline-external-thunk
+
+RETPOLINE_CFLAGS += $(call cc-option,$(RETPOLINE_CFLAGS_GCC),$(call 
cc-option,$(RETPOLINE_CFLAGS_CLANG)))
 ifneq ($(RETPOLINE_CFLAGS),)
 KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
 endif
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index d02a4df..d3f264a 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -27,3 +27,8 @@
 #if __has_feature(address_sanitizer)
 #define __SANITIZE_ADDRESS__
 #endif
+
+/* Clang doesn't have a way to turn it off per-function, yet. */
+#ifdef __noretpoline
+#undef __noretpoline
+#endif
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index 73bc63e..673fbf9 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -93,6 +93,10 @@
 #define __weak __attribute__((weak))
 #define __alias(symbol)__attribute__((alias(#symbol)))
 
+#ifdef RETPOLINE
+#define __noretpoline __attribute__((indirect_branch("keep")))
+#endif
+
 /*
  * it doesn't make sense on ARM (currently the only user of __naked)
  * to trace naked functions because then mcount is called without
diff --git a/include/linux/init.h b/include/linux/init.h
index 506a981..bc27cf0 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -6,10 +6,10 @@
 #include 
 
 /* Built-in __init functions needn't be compiled with retpoline */
-#if defined(RETPOLINE) && !defined(MODULE)
-#define __noretpoline __attribute__((indirect_branch("keep")))
+#if defined(__noretpoline) && !defined(MODULE)
+#define __noinitretpoline __noretpoline
 #else
-#define __noretpoline
+#define __noinitretpoline
 #endif
 
 /* These macros are used to mark some functions or 
@@ -47,7 +47,7 @@
 
 /* These are for everybody (although not all archs will actually
discard it in modules) */
-#define __init __section(.init.text) __cold  __latent_entropy 
__noretpoline
+#define __init __section(.init.text) __cold  __latent_entropy 
__noinitretpoline
 #define __initdata __section(.init.data)
 #define __initconst__section(.init.rodata)
 #define __exitdata __section(.exit.data)

[tip:x86/pti] x86/speculation: Use IBRS if available before calling into firmware

[tip-bot for David Woodhouse]

Commit-ID:  dd84441a797150dcc49298ec95c459a8891d8bb1
Gitweb: https://git.kernel.org/tip/dd84441a797150dcc49298ec95c459a8891d8bb1
Author: David Woodhouse 
AuthorDate: Mon, 19 Feb 2018 10:50:54 +
Committer:  Ingo Molnar 
CommitDate: Tue, 20 Feb 2018 09:38:33 +0100

x86/speculation: Use IBRS if available before calling into firmware

Retpoline means the kernel is safe because it has no indirect branches.
But firmware isn't, so use IBRS for firmware calls if it's available.

Block preemption while IBRS is set, although in practice the call sites
already had to be doing that.

Ignore hpwdt.c for now. It's taking spinlocks and calling into firmware
code, from an NMI handler. I don't want to touch that with a bargepole.

Signed-off-by: David Woodhouse 
Reviewed-by: Thomas Gleixner 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: arjan.van.de@intel.com
Cc: b...@alien8.de
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1519037457-7643-2-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/apm.h   |  6 ++
 arch/x86/include/asm/cpufeatures.h   |  1 +
 arch/x86/include/asm/efi.h   | 17 ++--
 arch/x86/include/asm/nospec-branch.h | 39 +++-
 arch/x86/kernel/cpu/bugs.c   | 12 ++-
 5 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 4d4015d..c356098 100644
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -7,6 +7,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include 
+
 #ifdef APM_ZERO_SEGS
 #  define APM_DO_ZERO_SEGS \
"pushl %%ds\n\t" \
@@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
  "=S" (*esi)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
 }
 
 static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
  "=S" (si)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
return error;
 }
 
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 0dfe4d3..f41079d 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -213,6 +213,7 @@
 #define X86_FEATURE_SEV( 7*32+20) /* AMD Secure 
Encrypted Virtualization */
 
 #define X86_FEATURE_USE_IBPB   ( 7*32+21) /* "" Indirect Branch 
Prediction Barrier enabled */
+#define X86_FEATURE_USE_IBRS_FW( 7*32+22) /* "" Use IBRS 
during runtime firmware calls */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 85f6ccb..a399c1e 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -6,6 +6,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -36,8 +37,18 @@
 
 extern asmlinkage unsigned long efi_call_phys(void *, ...);
 
-#define arch_efi_call_virt_setup() kernel_fpu_begin()
-#define arch_efi_call_virt_teardown()  kernel_fpu_end()
+#define arch_efi_call_virt_setup() \
+({ \
+   kernel_fpu_begin(); \
+   firmware_restrict_branch_speculation_start();   \
+})
+
+#define arch_efi_call_virt_teardown()  \
+({ \
+ 

[tip:x86/pti] x86/speculation: Use IBRS if available before calling into firmware

[tip-bot for David Woodhouse]

Commit-ID:  dd84441a797150dcc49298ec95c459a8891d8bb1
Gitweb: https://git.kernel.org/tip/dd84441a797150dcc49298ec95c459a8891d8bb1
Author: David Woodhouse 
AuthorDate: Mon, 19 Feb 2018 10:50:54 +
Committer:  Ingo Molnar 
CommitDate: Tue, 20 Feb 2018 09:38:33 +0100

x86/speculation: Use IBRS if available before calling into firmware

Retpoline means the kernel is safe because it has no indirect branches.
But firmware isn't, so use IBRS for firmware calls if it's available.

Block preemption while IBRS is set, although in practice the call sites
already had to be doing that.

Ignore hpwdt.c for now. It's taking spinlocks and calling into firmware
code, from an NMI handler. I don't want to touch that with a bargepole.

Signed-off-by: David Woodhouse 
Reviewed-by: Thomas Gleixner 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: arjan.van.de@intel.com
Cc: b...@alien8.de
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1519037457-7643-2-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/apm.h   |  6 ++
 arch/x86/include/asm/cpufeatures.h   |  1 +
 arch/x86/include/asm/efi.h   | 17 ++--
 arch/x86/include/asm/nospec-branch.h | 39 +++-
 arch/x86/kernel/cpu/bugs.c   | 12 ++-
 5 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 4d4015d..c356098 100644
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -7,6 +7,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include 
+
 #ifdef APM_ZERO_SEGS
 #  define APM_DO_ZERO_SEGS \
"pushl %%ds\n\t" \
@@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
  "=S" (*esi)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
 }
 
 static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
  "=S" (si)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
return error;
 }
 
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 0dfe4d3..f41079d 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -213,6 +213,7 @@
 #define X86_FEATURE_SEV( 7*32+20) /* AMD Secure 
Encrypted Virtualization */
 
 #define X86_FEATURE_USE_IBPB   ( 7*32+21) /* "" Indirect Branch 
Prediction Barrier enabled */
+#define X86_FEATURE_USE_IBRS_FW( 7*32+22) /* "" Use IBRS 
during runtime firmware calls */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 85f6ccb..a399c1e 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -6,6 +6,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -36,8 +37,18 @@
 
 extern asmlinkage unsigned long efi_call_phys(void *, ...);
 
-#define arch_efi_call_virt_setup() kernel_fpu_begin()
-#define arch_efi_call_virt_teardown()  kernel_fpu_end()
+#define arch_efi_call_virt_setup() \
+({ \
+   kernel_fpu_begin(); \
+   firmware_restrict_branch_speculation_start();   \
+})
+
+#define arch_efi_call_virt_teardown()  \
+({ \
+   firmware_restrict_branch_speculation_end(); \
+   kernel_fpu_end();   \

[tip:x86/pti] Revert "x86/retpoline: Simplify vmexit_fill_RSB()"

[tip-bot for David Woodhouse]

Commit-ID:  d1c99108af3c5992640aa2afa7d2e88c3775c06e
Gitweb: https://git.kernel.org/tip/d1c99108af3c5992640aa2afa7d2e88c3775c06e
Author: David Woodhouse 
AuthorDate: Mon, 19 Feb 2018 10:50:56 +
Committer:  Ingo Molnar 
CommitDate: Tue, 20 Feb 2018 09:38:26 +0100

Revert "x86/retpoline: Simplify vmexit_fill_RSB()"

This reverts commit 1dde7415e99933bb7293d6b2843752cbdb43ec11. By putting
the RSB filling out of line and calling it, we waste one RSB slot for
returning from the function itself, which means one fewer actual function
call we can make if we're doing the Skylake abomination of call-depth
counting.

It also changed the number of RSB stuffings we do on vmexit from 32,
which was correct, to 16. Let's just stop with the bikeshedding; it
didn't actually *fix* anything anyway.

Signed-off-by: David Woodhouse 
Acked-by: Thomas Gleixner 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: arjan.van.de@intel.com
Cc: b...@alien8.de
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1519037457-7643-4-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/entry/entry_32.S |  3 +-
 arch/x86/entry/entry_64.S |  3 +-
 arch/x86/include/asm/asm-prototypes.h |  3 --
 arch/x86/include/asm/nospec-branch.h  | 70 +++
 arch/x86/lib/Makefile |  1 -
 arch/x86/lib/retpoline.S  | 56 
 6 files changed, 65 insertions(+), 71 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 16c2c02..6ad064c 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -252,8 +252,7 @@ ENTRY(__switch_to_asm)
 * exist, overwrite the RSB with entries which capture
 * speculative execution to prevent attack.
 */
-   /* Clobbers %ebx */
-   FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+   FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
 #endif
 
/* restore callee-saved registers */
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 77edc23..7a53879 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -364,8 +364,7 @@ ENTRY(__switch_to_asm)
 * exist, overwrite the RSB with entries which capture
 * speculative execution to prevent attack.
 */
-   /* Clobbers %rbx */
-   FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+   FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
 #endif
 
/* restore callee-saved registers */
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index 4d11161..1908214 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -38,7 +38,4 @@ INDIRECT_THUNK(dx)
 INDIRECT_THUNK(si)
 INDIRECT_THUNK(di)
 INDIRECT_THUNK(bp)
-asmlinkage void __fill_rsb(void);
-asmlinkage void __clear_rsb(void);
-
 #endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 76b0585..af34b1e 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -8,6 +8,50 @@
 #include 
 #include 
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; lfence; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include ") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS32  /* To forcibly overwrite all 
entries */
+#define RSB_FILL_LOOPS 16  /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)  \
+   mov $(nr/2), reg;   \
+771:   \
+   call772f;   \
+773:   /* speculation trap */  \
+   pause;  \
+   lfence; \
+   

[tip:x86/pti] Revert "x86/retpoline: Simplify vmexit_fill_RSB()"

[tip-bot for David Woodhouse]

Commit-ID:  d1c99108af3c5992640aa2afa7d2e88c3775c06e
Gitweb: https://git.kernel.org/tip/d1c99108af3c5992640aa2afa7d2e88c3775c06e
Author: David Woodhouse 
AuthorDate: Mon, 19 Feb 2018 10:50:56 +
Committer:  Ingo Molnar 
CommitDate: Tue, 20 Feb 2018 09:38:26 +0100

Revert "x86/retpoline: Simplify vmexit_fill_RSB()"

This reverts commit 1dde7415e99933bb7293d6b2843752cbdb43ec11. By putting
the RSB filling out of line and calling it, we waste one RSB slot for
returning from the function itself, which means one fewer actual function
call we can make if we're doing the Skylake abomination of call-depth
counting.

It also changed the number of RSB stuffings we do on vmexit from 32,
which was correct, to 16. Let's just stop with the bikeshedding; it
didn't actually *fix* anything anyway.

Signed-off-by: David Woodhouse 
Acked-by: Thomas Gleixner 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: arjan.van.de@intel.com
Cc: b...@alien8.de
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1519037457-7643-4-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/entry/entry_32.S |  3 +-
 arch/x86/entry/entry_64.S |  3 +-
 arch/x86/include/asm/asm-prototypes.h |  3 --
 arch/x86/include/asm/nospec-branch.h  | 70 +++
 arch/x86/lib/Makefile |  1 -
 arch/x86/lib/retpoline.S  | 56 
 6 files changed, 65 insertions(+), 71 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 16c2c02..6ad064c 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -252,8 +252,7 @@ ENTRY(__switch_to_asm)
 * exist, overwrite the RSB with entries which capture
 * speculative execution to prevent attack.
 */
-   /* Clobbers %ebx */
-   FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+   FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
 #endif
 
/* restore callee-saved registers */
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 77edc23..7a53879 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -364,8 +364,7 @@ ENTRY(__switch_to_asm)
 * exist, overwrite the RSB with entries which capture
 * speculative execution to prevent attack.
 */
-   /* Clobbers %rbx */
-   FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+   FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
 #endif
 
/* restore callee-saved registers */
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index 4d11161..1908214 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -38,7 +38,4 @@ INDIRECT_THUNK(dx)
 INDIRECT_THUNK(si)
 INDIRECT_THUNK(di)
 INDIRECT_THUNK(bp)
-asmlinkage void __fill_rsb(void);
-asmlinkage void __clear_rsb(void);
-
 #endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 76b0585..af34b1e 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -8,6 +8,50 @@
 #include 
 #include 
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; lfence; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include ") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS32  /* To forcibly overwrite all 
entries */
+#define RSB_FILL_LOOPS 16  /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)  \
+   mov $(nr/2), reg;   \
+771:   \
+   call772f;   \
+773:   /* speculation trap */  \
+   pause;  \
+   lfence; \
+   jmp 773b;   \
+772:   \
+   call774f;   \
+775:   

[tip:x86/pti] Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

[tip-bot for David Woodhouse]

Commit-ID:  f208820a321f9b23d77d7eed89945d862d62a3ed
Gitweb: https://git.kernel.org/tip/f208820a321f9b23d77d7eed89945d862d62a3ed
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:23 +
Committer:  Ingo Molnar 
CommitDate: Tue, 13 Feb 2018 08:59:00 +0100

Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

This reverts commit 64e16720ea0879f8ab4547e3b9758936d483909b.

We cannot call C functions like that, without marking all the
call-clobbered registers as, well, clobbered. We might have got away
with it for now because the __ibp_barrier() function was *fairly*
unlikely to actually use any other registers. But no. Just no.

Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Cc: sir...@amazon.de
Link: 
http://lkml.kernel.org/r/1518305967-31356-3-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/nospec-branch.h | 13 +
 arch/x86/include/asm/processor.h |  3 ---
 arch/x86/kernel/cpu/bugs.c   |  6 --
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 4d57894..300cc15 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -164,10 +164,15 @@ static inline void vmexit_fill_RSB(void)
 
 static inline void indirect_branch_prediction_barrier(void)
 {
-   alternative_input("",
- "call __ibp_barrier",
- X86_FEATURE_USE_IBPB,
- ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory"));
+   asm volatile(ALTERNATIVE("",
+"movl %[msr], %%ecx\n\t"
+"movl %[val], %%eax\n\t"
+"movl $0, %%edx\n\t"
+"wrmsr",
+X86_FEATURE_USE_IBPB)
+: : [msr] "i" (MSR_IA32_PRED_CMD),
+[val] "i" (PRED_CMD_IBPB)
+: "eax", "ecx", "edx", "memory");
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 513f960..99799fb 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -969,7 +969,4 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
-
-void __ibp_barrier(void);
-
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 71949bf..61152aa 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -337,9 +337,3 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
   spectre_v2_module_string());
 }
 #endif
-
-void __ibp_barrier(void)
-{
-   __wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0);
-}
-EXPORT_SYMBOL_GPL(__ibp_barrier);

[tip:x86/pti] Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

[tip-bot for David Woodhouse]

Commit-ID:  f208820a321f9b23d77d7eed89945d862d62a3ed
Gitweb: https://git.kernel.org/tip/f208820a321f9b23d77d7eed89945d862d62a3ed
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:23 +
Committer:  Ingo Molnar 
CommitDate: Tue, 13 Feb 2018 08:59:00 +0100

Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

This reverts commit 64e16720ea0879f8ab4547e3b9758936d483909b.

We cannot call C functions like that, without marking all the
call-clobbered registers as, well, clobbered. We might have got away
with it for now because the __ibp_barrier() function was *fairly*
unlikely to actually use any other registers. But no. Just no.

Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Cc: sir...@amazon.de
Link: 
http://lkml.kernel.org/r/1518305967-31356-3-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/nospec-branch.h | 13 +
 arch/x86/include/asm/processor.h |  3 ---
 arch/x86/kernel/cpu/bugs.c   |  6 --
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 4d57894..300cc15 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -164,10 +164,15 @@ static inline void vmexit_fill_RSB(void)
 
 static inline void indirect_branch_prediction_barrier(void)
 {
-   alternative_input("",
- "call __ibp_barrier",
- X86_FEATURE_USE_IBPB,
- ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory"));
+   asm volatile(ALTERNATIVE("",
+"movl %[msr], %%ecx\n\t"
+"movl %[val], %%eax\n\t"
+"movl $0, %%edx\n\t"
+"wrmsr",
+X86_FEATURE_USE_IBPB)
+: : [msr] "i" (MSR_IA32_PRED_CMD),
+[val] "i" (PRED_CMD_IBPB)
+: "eax", "ecx", "edx", "memory");
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 513f960..99799fb 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -969,7 +969,4 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
-
-void __ibp_barrier(void);
-
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 71949bf..61152aa 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -337,9 +337,3 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
   spectre_v2_module_string());
 }
 #endif
-
-void __ibp_barrier(void)
-{
-   __wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0);
-}
-EXPORT_SYMBOL_GPL(__ibp_barrier);

[tip:x86/pti] KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods

[tip-bot for David Woodhouse]

Commit-ID:  928a4c39484281f8ca366f53a1db79330d058401
Gitweb: https://git.kernel.org/tip/928a4c39484281f8ca366f53a1db79330d058401
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:24 +
Committer:  Ingo Molnar 
CommitDate: Tue, 13 Feb 2018 08:59:45 +0100

KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by 
always inlining iterator helper methods

With retpoline, tight loops of "call this function for every XXX" are
very much pessimised by taking a prediction miss *every* time. This one
is by far the biggest contributor to the guest launch time with retpoline.

By marking the iterator slot_handle_…() functions always_inline, we can
ensure that the indirect function call can be optimised away into a
direct call and it actually generates slightly smaller code because
some of the other conditionals can get optimised away too.

Performance is now pretty close to what we see with nospectre_v2 on
the command line.

Suggested-by: Linus Torvalds 
Tested-by: Filippo Sironi 
Signed-off-by: David Woodhouse 
Reviewed-by: Filippo Sironi 
Acked-by: Paolo Bonzini 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kvm/mmu.c | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2b8eb4d..cc83bdc 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -5058,7 +5058,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head 
*rmap_head);
 
 /* The caller should hold mmu-lock before calling this function. */
-static bool
+static __always_inline bool
 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, int start_level, int end_level,
gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -5088,7 +5088,7 @@ slot_handle_level_range(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
return flush;
 }
 
-static bool
+static __always_inline bool
 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, int start_level, int end_level,
  bool lock_flush_tlb)
@@ -5099,7 +5099,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot 
*memslot,
lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5107,7 +5107,7 @@ slot_handle_all_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5115,7 +5115,7 @@ slot_handle_large_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
 slot_level_handler fn, bool lock_flush_tlb)
 {

[tip:x86/pti] KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods

[tip-bot for David Woodhouse]

Commit-ID:  928a4c39484281f8ca366f53a1db79330d058401
Gitweb: https://git.kernel.org/tip/928a4c39484281f8ca366f53a1db79330d058401
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:24 +
Committer:  Ingo Molnar 
CommitDate: Tue, 13 Feb 2018 08:59:45 +0100

KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by 
always inlining iterator helper methods

With retpoline, tight loops of "call this function for every XXX" are
very much pessimised by taking a prediction miss *every* time. This one
is by far the biggest contributor to the guest launch time with retpoline.

By marking the iterator slot_handle_…() functions always_inline, we can
ensure that the indirect function call can be optimised away into a
direct call and it actually generates slightly smaller code because
some of the other conditionals can get optimised away too.

Performance is now pretty close to what we see with nospectre_v2 on
the command line.

Suggested-by: Linus Torvalds 
Tested-by: Filippo Sironi 
Signed-off-by: David Woodhouse 
Reviewed-by: Filippo Sironi 
Acked-by: Paolo Bonzini 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kvm/mmu.c | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2b8eb4d..cc83bdc 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -5058,7 +5058,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head 
*rmap_head);
 
 /* The caller should hold mmu-lock before calling this function. */
-static bool
+static __always_inline bool
 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, int start_level, int end_level,
gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -5088,7 +5088,7 @@ slot_handle_level_range(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
return flush;
 }
 
-static bool
+static __always_inline bool
 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, int start_level, int end_level,
  bool lock_flush_tlb)
@@ -5099,7 +5099,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot 
*memslot,
lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5107,7 +5107,7 @@ slot_handle_all_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5115,7 +5115,7 @@ slot_handle_large_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
 slot_level_handler fn, bool lock_flush_tlb)
 {

[tip:x86/pti] x86/speculation: Correct Speculation Control microcode blacklist again

[tip-bot for David Woodhouse]

Commit-ID:  d37fc6d360a404b208547ba112e7dabb6533c7fc
Gitweb: https://git.kernel.org/tip/d37fc6d360a404b208547ba112e7dabb6533c7fc
Author: David Woodhouse 
AuthorDate: Mon, 12 Feb 2018 15:27:34 +
Committer:  Ingo Molnar 
CommitDate: Tue, 13 Feb 2018 08:58:59 +0100

x86/speculation: Correct Speculation Control microcode blacklist again

Arjan points out that the Intel document only clears the 0xc2 microcode
on *some* parts with CPUID 506E3 (INTEL_FAM6_SKYLAKE_DESKTOP stepping 3).
For the Skylake H/S platform it's OK but for Skylake E3 which has the
same CPUID it isn't (yet) cleared.

So removing it from the blacklist was premature. Put it back for now.

Also, Arjan assures me that the 0x84 microcode for Kaby Lake which was
featured in one of the early revisions of the Intel document was never
released to the public, and won't be until/unless it is also validated
as safe. So those can change to 0x80 which is what all *other* versions
of the doc have identified.

Once the retrospective testing of existing public microcodes is done, we
should be back into a mode where new microcodes are only released in
batches and we shouldn't even need to update the blacklist for those
anyway, so this tweaking of the list isn't expected to be a thing which
keeps happening.

Requested-by: Arjan van de Ven 
Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Link: 
http://lkml.kernel.org/r/1518449255-2182-1-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kernel/cpu/intel.c | 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index f73b814..ef796f1 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -116,13 +116,14 @@ struct sku_microcode {
u32 microcode;
 };
 static const struct sku_microcode spectre_bad_microcodes[] = {
-   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0B,   0x84 },
-   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0A,   0x84 },
-   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x09,   0x84 },
-   { INTEL_FAM6_KABYLAKE_MOBILE,   0x0A,   0x84 },
-   { INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x84 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0B,   0x80 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0A,   0x80 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x09,   0x80 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x0A,   0x80 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x80 },
{ INTEL_FAM6_SKYLAKE_X, 0x03,   0x0100013e },
{ INTEL_FAM6_SKYLAKE_X, 0x04,   0x023c },
+   { INTEL_FAM6_SKYLAKE_DESKTOP,   0x03,   0xc2 },
{ INTEL_FAM6_BROADWELL_CORE,0x04,   0x28 },
{ INTEL_FAM6_BROADWELL_GT3E,0x01,   0x1b },
{ INTEL_FAM6_BROADWELL_XEON_D,  0x02,   0x14 },

[tip:x86/pti] x86/speculation: Correct Speculation Control microcode blacklist again

[tip-bot for David Woodhouse]

Commit-ID:  d37fc6d360a404b208547ba112e7dabb6533c7fc
Gitweb: https://git.kernel.org/tip/d37fc6d360a404b208547ba112e7dabb6533c7fc
Author: David Woodhouse 
AuthorDate: Mon, 12 Feb 2018 15:27:34 +
Committer:  Ingo Molnar 
CommitDate: Tue, 13 Feb 2018 08:58:59 +0100

x86/speculation: Correct Speculation Control microcode blacklist again

Arjan points out that the Intel document only clears the 0xc2 microcode
on *some* parts with CPUID 506E3 (INTEL_FAM6_SKYLAKE_DESKTOP stepping 3).
For the Skylake H/S platform it's OK but for Skylake E3 which has the
same CPUID it isn't (yet) cleared.

So removing it from the blacklist was premature. Put it back for now.

Also, Arjan assures me that the 0x84 microcode for Kaby Lake which was
featured in one of the early revisions of the Intel document was never
released to the public, and won't be until/unless it is also validated
as safe. So those can change to 0x80 which is what all *other* versions
of the doc have identified.

Once the retrospective testing of existing public microcodes is done, we
should be back into a mode where new microcodes are only released in
batches and we shouldn't even need to update the blacklist for those
anyway, so this tweaking of the list isn't expected to be a thing which
keeps happening.

Requested-by: Arjan van de Ven 
Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Link: 
http://lkml.kernel.org/r/1518449255-2182-1-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kernel/cpu/intel.c | 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index f73b814..ef796f1 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -116,13 +116,14 @@ struct sku_microcode {
u32 microcode;
 };
 static const struct sku_microcode spectre_bad_microcodes[] = {
-   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0B,   0x84 },
-   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0A,   0x84 },
-   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x09,   0x84 },
-   { INTEL_FAM6_KABYLAKE_MOBILE,   0x0A,   0x84 },
-   { INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x84 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0B,   0x80 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0A,   0x80 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x09,   0x80 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x0A,   0x80 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x80 },
{ INTEL_FAM6_SKYLAKE_X, 0x03,   0x0100013e },
{ INTEL_FAM6_SKYLAKE_X, 0x04,   0x023c },
+   { INTEL_FAM6_SKYLAKE_DESKTOP,   0x03,   0xc2 },
{ INTEL_FAM6_BROADWELL_CORE,0x04,   0x28 },
{ INTEL_FAM6_BROADWELL_GT3E,0x01,   0x1b },
{ INTEL_FAM6_BROADWELL_XEON_D,  0x02,   0x14 },

[tip:x86/pti] Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

[tip-bot for David Woodhouse]

Commit-ID:  930ce1a7a55bc0eb8917f453ee22f1b6d67df5cd
Gitweb: https://git.kernel.org/tip/930ce1a7a55bc0eb8917f453ee22f1b6d67df5cd
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:23 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 11:24:15 +0100

Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

This reverts commit 64e16720ea0879f8ab4547e3b9758936d483909b.

We cannot call C functions like that, without marking all the
call-clobbered registers as, well, clobbered. We might have got away
with it for now because the __ibp_barrier() function was *fairly*
unlikely to actually use any other registers. But no. Just no.

Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Cc: sir...@amazon.de
Link: 
http://lkml.kernel.org/r/1518305967-31356-3-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/nospec-branch.h | 13 +
 arch/x86/include/asm/processor.h |  3 ---
 arch/x86/kernel/cpu/bugs.c   |  6 --
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 4d57894..300cc15 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -164,10 +164,15 @@ static inline void vmexit_fill_RSB(void)
 
 static inline void indirect_branch_prediction_barrier(void)
 {
-   alternative_input("",
- "call __ibp_barrier",
- X86_FEATURE_USE_IBPB,
- ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory"));
+   asm volatile(ALTERNATIVE("",
+"movl %[msr], %%ecx\n\t"
+"movl %[val], %%eax\n\t"
+"movl $0, %%edx\n\t"
+"wrmsr",
+X86_FEATURE_USE_IBPB)
+: : [msr] "i" (MSR_IA32_PRED_CMD),
+[val] "i" (PRED_CMD_IBPB)
+: "eax", "ecx", "edx", "memory");
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 513f960..99799fb 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -969,7 +969,4 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
-
-void __ibp_barrier(void);
-
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 71949bf..61152aa 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -337,9 +337,3 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
   spectre_v2_module_string());
 }
 #endif
-
-void __ibp_barrier(void)
-{
-   __wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0);
-}
-EXPORT_SYMBOL_GPL(__ibp_barrier);

[tip:x86/pti] Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

[tip-bot for David Woodhouse]

Commit-ID:  930ce1a7a55bc0eb8917f453ee22f1b6d67df5cd
Gitweb: https://git.kernel.org/tip/930ce1a7a55bc0eb8917f453ee22f1b6d67df5cd
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:23 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 11:24:15 +0100

Revert "x86/speculation: Simplify indirect_branch_prediction_barrier()"

This reverts commit 64e16720ea0879f8ab4547e3b9758936d483909b.

We cannot call C functions like that, without marking all the
call-clobbered registers as, well, clobbered. We might have got away
with it for now because the __ibp_barrier() function was *fairly*
unlikely to actually use any other registers. But no. Just no.

Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Cc: sir...@amazon.de
Link: 
http://lkml.kernel.org/r/1518305967-31356-3-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/nospec-branch.h | 13 +
 arch/x86/include/asm/processor.h |  3 ---
 arch/x86/kernel/cpu/bugs.c   |  6 --
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 4d57894..300cc15 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -164,10 +164,15 @@ static inline void vmexit_fill_RSB(void)
 
 static inline void indirect_branch_prediction_barrier(void)
 {
-   alternative_input("",
- "call __ibp_barrier",
- X86_FEATURE_USE_IBPB,
- ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory"));
+   asm volatile(ALTERNATIVE("",
+"movl %[msr], %%ecx\n\t"
+"movl %[val], %%eax\n\t"
+"movl $0, %%edx\n\t"
+"wrmsr",
+X86_FEATURE_USE_IBPB)
+: : [msr] "i" (MSR_IA32_PRED_CMD),
+[val] "i" (PRED_CMD_IBPB)
+: "eax", "ecx", "edx", "memory");
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 513f960..99799fb 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -969,7 +969,4 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
-
-void __ibp_barrier(void);
-
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 71949bf..61152aa 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -337,9 +337,3 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
   spectre_v2_module_string());
 }
 #endif
-
-void __ibp_barrier(void)
-{
-   __wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0);
-}
-EXPORT_SYMBOL_GPL(__ibp_barrier);

[tip:x86/pti] KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods

[tip-bot for David Woodhouse]

Commit-ID:  33f1e899478efb7c77b2b833e7edee1203a24a48
Gitweb: https://git.kernel.org/tip/33f1e899478efb7c77b2b833e7edee1203a24a48
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:24 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 11:24:15 +0100

KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by 
always inlining iterator helper methods

With retpoline, tight loops of "call this function for every XXX" are
very much pessimised by taking a prediction miss *every* time. This one
is by far the biggest contributor to the guest launch time with retpoline.

By marking the iterator slot_handle_…() functions always_inline, we can
ensure that the indirect function call can be optimised away into a
direct call and it actually generates slightly smaller code because
some of the other conditionals can get optimised away too.

Performance is now pretty close to what we see with nospectre_v2 on
the command line.

Suggested-by: Linus Torvalds 
Tested-by: Filippo Sironi 
Signed-off-by: David Woodhouse 
Reviewed-by: Filippo Sironi 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kvm/mmu.c | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2b8eb4d..cc83bdc 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -5058,7 +5058,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head 
*rmap_head);
 
 /* The caller should hold mmu-lock before calling this function. */
-static bool
+static __always_inline bool
 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, int start_level, int end_level,
gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -5088,7 +5088,7 @@ slot_handle_level_range(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
return flush;
 }
 
-static bool
+static __always_inline bool
 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, int start_level, int end_level,
  bool lock_flush_tlb)
@@ -5099,7 +5099,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot 
*memslot,
lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5107,7 +5107,7 @@ slot_handle_all_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5115,7 +5115,7 @@ slot_handle_large_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
 slot_level_handler fn, bool lock_flush_tlb)
 {

[tip:x86/pti] KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods

[tip-bot for David Woodhouse]

Commit-ID:  33f1e899478efb7c77b2b833e7edee1203a24a48
Gitweb: https://git.kernel.org/tip/33f1e899478efb7c77b2b833e7edee1203a24a48
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:24 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 11:24:15 +0100

KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by 
always inlining iterator helper methods

With retpoline, tight loops of "call this function for every XXX" are
very much pessimised by taking a prediction miss *every* time. This one
is by far the biggest contributor to the guest launch time with retpoline.

By marking the iterator slot_handle_…() functions always_inline, we can
ensure that the indirect function call can be optimised away into a
direct call and it actually generates slightly smaller code because
some of the other conditionals can get optimised away too.

Performance is now pretty close to what we see with nospectre_v2 on
the command line.

Suggested-by: Linus Torvalds 
Tested-by: Filippo Sironi 
Signed-off-by: David Woodhouse 
Reviewed-by: Filippo Sironi 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: dave.han...@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Link: 
http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kvm/mmu.c | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2b8eb4d..cc83bdc 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -5058,7 +5058,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head 
*rmap_head);
 
 /* The caller should hold mmu-lock before calling this function. */
-static bool
+static __always_inline bool
 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, int start_level, int end_level,
gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -5088,7 +5088,7 @@ slot_handle_level_range(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
return flush;
 }
 
-static bool
+static __always_inline bool
 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, int start_level, int end_level,
  bool lock_flush_tlb)
@@ -5099,7 +5099,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot 
*memslot,
lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5107,7 +5107,7 @@ slot_handle_all_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5115,7 +5115,7 @@ slot_handle_large_level(struct kvm *kvm, struct 
kvm_memory_slot *memslot,
 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
 slot_level_handler fn, bool lock_flush_tlb)
 {

[tip:x86/pti] x86/speculation: Update Speculation Control microcode blacklist

[tip-bot for David Woodhouse]

Commit-ID:  1751342095f0d2b36fa8114d8e12c5688c455ac4
Gitweb: https://git.kernel.org/tip/1751342095f0d2b36fa8114d8e12c5688c455ac4
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:22 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 11:24:15 +0100

x86/speculation: Update Speculation Control microcode blacklist

Intel have retroactively blessed the 0xc2 microcode on Skylake mobile
and desktop parts, and the Gemini Lake 0x22 microcode is apparently fine
too. We blacklisted the latter purely because it was present with all
the other problematic ones in the 2018-01-08 release, but now it's
explicitly listed as OK.

We still list 0x84 for the various Kaby Lake / Coffee Lake parts, as
that appeared in one version of the blacklist and then reverted to
0x80 again. We can change it if 0x84 is actually announced to be safe.

Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Cc: sir...@amazon.de
Link: 
http://lkml.kernel.org/r/1518305967-31356-2-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kernel/cpu/intel.c | 4 
 1 file changed, 4 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 319bf98..f73b814 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -123,8 +123,6 @@ static const struct sku_microcode spectre_bad_microcodes[] 
= {
{ INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x84 },
{ INTEL_FAM6_SKYLAKE_X, 0x03,   0x0100013e },
{ INTEL_FAM6_SKYLAKE_X, 0x04,   0x023c },
-   { INTEL_FAM6_SKYLAKE_MOBILE,0x03,   0xc2 },
-   { INTEL_FAM6_SKYLAKE_DESKTOP,   0x03,   0xc2 },
{ INTEL_FAM6_BROADWELL_CORE,0x04,   0x28 },
{ INTEL_FAM6_BROADWELL_GT3E,0x01,   0x1b },
{ INTEL_FAM6_BROADWELL_XEON_D,  0x02,   0x14 },
@@ -136,8 +134,6 @@ static const struct sku_microcode spectre_bad_microcodes[] 
= {
{ INTEL_FAM6_HASWELL_X, 0x02,   0x3b },
{ INTEL_FAM6_HASWELL_X, 0x04,   0x10 },
{ INTEL_FAM6_IVYBRIDGE_X,   0x04,   0x42a },
-   /* Updated in the 20180108 release; blacklist until we know otherwise */
-   { INTEL_FAM6_ATOM_GEMINI_LAKE,  0x01,   0x22 },
/* Observed in the wild */
{ INTEL_FAM6_SANDYBRIDGE_X, 0x06,   0x61b },
{ INTEL_FAM6_SANDYBRIDGE_X, 0x07,   0x712 },

[tip:x86/pti] x86/speculation: Update Speculation Control microcode blacklist

[tip-bot for David Woodhouse]

Commit-ID:  1751342095f0d2b36fa8114d8e12c5688c455ac4
Gitweb: https://git.kernel.org/tip/1751342095f0d2b36fa8114d8e12c5688c455ac4
Author: David Woodhouse 
AuthorDate: Sat, 10 Feb 2018 23:39:22 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 11:24:15 +0100

x86/speculation: Update Speculation Control microcode blacklist

Intel have retroactively blessed the 0xc2 microcode on Skylake mobile
and desktop parts, and the Gemini Lake 0x22 microcode is apparently fine
too. We blacklisted the latter purely because it was present with all
the other problematic ones in the 2018-01-08 release, but now it's
explicitly listed as OK.

We still list 0x84 for the various Kaby Lake / Coffee Lake parts, as
that appeared in one version of the blacklist and then reverted to
0x80 again. We can change it if 0x84 is actually announced to be safe.

Signed-off-by: David Woodhouse 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: Dave Hansen 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: arjan.van.de@intel.com
Cc: jmatt...@google.com
Cc: karah...@amazon.de
Cc: k...@vger.kernel.org
Cc: pbonz...@redhat.com
Cc: rkrc...@redhat.com
Cc: sir...@amazon.de
Link: 
http://lkml.kernel.org/r/1518305967-31356-2-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/kernel/cpu/intel.c | 4 
 1 file changed, 4 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 319bf98..f73b814 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -123,8 +123,6 @@ static const struct sku_microcode spectre_bad_microcodes[] 
= {
{ INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x84 },
{ INTEL_FAM6_SKYLAKE_X, 0x03,   0x0100013e },
{ INTEL_FAM6_SKYLAKE_X, 0x04,   0x023c },
-   { INTEL_FAM6_SKYLAKE_MOBILE,0x03,   0xc2 },
-   { INTEL_FAM6_SKYLAKE_DESKTOP,   0x03,   0xc2 },
{ INTEL_FAM6_BROADWELL_CORE,0x04,   0x28 },
{ INTEL_FAM6_BROADWELL_GT3E,0x01,   0x1b },
{ INTEL_FAM6_BROADWELL_XEON_D,  0x02,   0x14 },
@@ -136,8 +134,6 @@ static const struct sku_microcode spectre_bad_microcodes[] 
= {
{ INTEL_FAM6_HASWELL_X, 0x02,   0x3b },
{ INTEL_FAM6_HASWELL_X, 0x04,   0x10 },
{ INTEL_FAM6_IVYBRIDGE_X,   0x04,   0x42a },
-   /* Updated in the 20180108 release; blacklist until we know otherwise */
-   { INTEL_FAM6_ATOM_GEMINI_LAKE,  0x01,   0x22 },
/* Observed in the wild */
{ INTEL_FAM6_SANDYBRIDGE_X, 0x06,   0x61b },
{ INTEL_FAM6_SANDYBRIDGE_X, 0x07,   0x712 },

[tip:x86/pti] x86/speculation: Use IBRS if available before calling into firmware

[tip-bot for David Woodhouse]

Commit-ID:  670c3e8da87fa4046a55077b1409cf250865a203
Gitweb: https://git.kernel.org/tip/670c3e8da87fa4046a55077b1409cf250865a203
Author: David Woodhouse 
AuthorDate: Sun, 11 Feb 2018 15:19:19 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 19:44:46 +0100

x86/speculation: Use IBRS if available before calling into firmware

Retpoline means the kernel is safe because it has no indirect branches.
But firmware isn't, so use IBRS for firmware calls if it's available.

Signed-off-by: David Woodhouse 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Link: 
http://lkml.kernel.org/r/1518362359-1005-1-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/apm.h   |  6 ++
 arch/x86/include/asm/cpufeatures.h   |  1 +
 arch/x86/include/asm/efi.h   | 17 +++--
 arch/x86/include/asm/nospec-branch.h | 37 +++-
 arch/x86/kernel/cpu/bugs.c   | 12 +++-
 drivers/watchdog/hpwdt.c |  3 +++
 6 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 4d4015d..c356098 100644
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -7,6 +7,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include 
+
 #ifdef APM_ZERO_SEGS
 #  define APM_DO_ZERO_SEGS \
"pushl %%ds\n\t" \
@@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
  "=S" (*esi)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
 }
 
 static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
  "=S" (si)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
return error;
 }
 
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 73b5fff..66c1434 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 #define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* "" Fill RSB on context 
switches */
 
 #define X86_FEATURE_USE_IBPB   ( 7*32+21) /* "" Indirect Branch 
Prediction Barrier enabled */
+#define X86_FEATURE_USE_IBRS_FW( 7*32+22) /* "" Use IBRS 
during runtime firmware calls */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 85f6ccb..a399c1e 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -6,6 +6,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -36,8 +37,18 @@
 
 extern asmlinkage unsigned long efi_call_phys(void *, ...);
 
-#define arch_efi_call_virt_setup() kernel_fpu_begin()
-#define arch_efi_call_virt_teardown()  kernel_fpu_end()
+#define arch_efi_call_virt_setup() \
+({ \
+   kernel_fpu_begin(); \
+   firmware_restrict_branch_speculation_start();   \
+})
+
+#define arch_efi_call_virt_teardown()  \
+({ \
+   firmware_restrict_branch_speculation_end(); \
+   kernel_fpu_end();   \
+})
+
 
 /*
  * Wrap all the virtual calls in a way that forces the parameters on the stack.
@@ -73,6 +84,7 @@ struct efi_scratch {
efi_sync_low_kernel_mappings(); \
preempt_disable(); 

[tip:x86/pti] x86/speculation: Use IBRS if available before calling into firmware

[tip-bot for David Woodhouse]

Commit-ID:  670c3e8da87fa4046a55077b1409cf250865a203
Gitweb: https://git.kernel.org/tip/670c3e8da87fa4046a55077b1409cf250865a203
Author: David Woodhouse 
AuthorDate: Sun, 11 Feb 2018 15:19:19 +
Committer:  Ingo Molnar 
CommitDate: Sun, 11 Feb 2018 19:44:46 +0100

x86/speculation: Use IBRS if available before calling into firmware

Retpoline means the kernel is safe because it has no indirect branches.
But firmware isn't, so use IBRS for firmware calls if it's available.

Signed-off-by: David Woodhouse 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Link: 
http://lkml.kernel.org/r/1518362359-1005-1-git-send-email-d...@amazon.co.uk
Signed-off-by: Ingo Molnar 
---
 arch/x86/include/asm/apm.h   |  6 ++
 arch/x86/include/asm/cpufeatures.h   |  1 +
 arch/x86/include/asm/efi.h   | 17 +++--
 arch/x86/include/asm/nospec-branch.h | 37 +++-
 arch/x86/kernel/cpu/bugs.c   | 12 +++-
 drivers/watchdog/hpwdt.c |  3 +++
 6 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 4d4015d..c356098 100644
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -7,6 +7,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include 
+
 #ifdef APM_ZERO_SEGS
 #  define APM_DO_ZERO_SEGS \
"pushl %%ds\n\t" \
@@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, 
u32 ecx_in,
  "=S" (*esi)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
 }
 
 static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
 * N.B. We do NOT need a cld after the BIOS call
 * because we always save and restore the flags.
 */
+   firmware_restrict_branch_speculation_start();
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
@@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 
ebx_in,
  "=S" (si)
: "a" (func), "b" (ebx_in), "c" (ecx_in)
: "memory", "cc");
+   firmware_restrict_branch_speculation_end();
return error;
 }
 
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 73b5fff..66c1434 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 #define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* "" Fill RSB on context 
switches */
 
 #define X86_FEATURE_USE_IBPB   ( 7*32+21) /* "" Indirect Branch 
Prediction Barrier enabled */
+#define X86_FEATURE_USE_IBRS_FW( 7*32+22) /* "" Use IBRS 
during runtime firmware calls */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 85f6ccb..a399c1e 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -6,6 +6,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -36,8 +37,18 @@
 
 extern asmlinkage unsigned long efi_call_phys(void *, ...);
 
-#define arch_efi_call_virt_setup() kernel_fpu_begin()
-#define arch_efi_call_virt_teardown()  kernel_fpu_end()
+#define arch_efi_call_virt_setup() \
+({ \
+   kernel_fpu_begin(); \
+   firmware_restrict_branch_speculation_start();   \
+})
+
+#define arch_efi_call_virt_teardown()  \
+({ \
+   firmware_restrict_branch_speculation_end(); \
+   kernel_fpu_end();   \
+})
+
 
 /*
  * Wrap all the virtual calls in a way that forces the parameters on the stack.
@@ -73,6 +84,7 @@ struct efi_scratch {
efi_sync_low_kernel_mappings(); \
preempt_disable();  \
__kernel_fpu_begin();   \
+   firmware_restrict_branch_speculation_start(); 

[tip:x86/pti] x86/retpoline: Avoid retpolines for built-in __init functions

[tip-bot for David Woodhouse]

Commit-ID:  66f793099a636862a71c59d4a6ba91387b155e0c
Gitweb: https://git.kernel.org/tip/66f793099a636862a71c59d4a6ba91387b155e0c
Author: David Woodhouse 
AuthorDate: Thu, 1 Feb 2018 11:27:20 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 2 Feb 2018 12:28:27 +0100

x86/retpoline: Avoid retpolines for built-in __init functions

There's no point in building init code with retpolines, since it runs before
any potentially hostile userspace does. And before the retpoline is actually
ALTERNATIVEd into place, for much of it.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: karah...@amazon.de
Cc: pet...@infradead.org
Cc: b...@alien8.de
Link: 
https://lkml.kernel.org/r/1517484441-1420-2-git-send-email-d...@amazon.co.uk

---
 include/linux/init.h | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/include/linux/init.h b/include/linux/init.h
index ea1b311..506a981 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -5,6 +5,13 @@
 #include 
 #include 
 
+/* Built-in __init functions needn't be compiled with retpoline */
+#if defined(RETPOLINE) && !defined(MODULE)
+#define __noretpoline __attribute__((indirect_branch("keep")))
+#else
+#define __noretpoline
+#endif
+
 /* These macros are used to mark some functions or 
  * initialized data (doesn't apply to uninitialized data)
  * as `initialization' functions. The kernel can take this
@@ -40,7 +47,7 @@
 
 /* These are for everybody (although not all archs will actually
discard it in modules) */
-#define __init __section(.init.text) __cold  __latent_entropy
+#define __init __section(.init.text) __cold  __latent_entropy 
__noretpoline
 #define __initdata __section(.init.data)
 #define __initconst__section(.init.rodata)
 #define __exitdata __section(.exit.data)

[tip:x86/pti] x86/retpoline: Avoid retpolines for built-in __init functions

[tip-bot for David Woodhouse]

Commit-ID:  66f793099a636862a71c59d4a6ba91387b155e0c
Gitweb: https://git.kernel.org/tip/66f793099a636862a71c59d4a6ba91387b155e0c
Author: David Woodhouse 
AuthorDate: Thu, 1 Feb 2018 11:27:20 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 2 Feb 2018 12:28:27 +0100

x86/retpoline: Avoid retpolines for built-in __init functions

There's no point in building init code with retpolines, since it runs before
any potentially hostile userspace does. And before the retpoline is actually
ALTERNATIVEd into place, for much of it.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: karah...@amazon.de
Cc: pet...@infradead.org
Cc: b...@alien8.de
Link: 
https://lkml.kernel.org/r/1517484441-1420-2-git-send-email-d...@amazon.co.uk

---
 include/linux/init.h | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/include/linux/init.h b/include/linux/init.h
index ea1b311..506a981 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -5,6 +5,13 @@
 #include 
 #include 
 
+/* Built-in __init functions needn't be compiled with retpoline */
+#if defined(RETPOLINE) && !defined(MODULE)
+#define __noretpoline __attribute__((indirect_branch("keep")))
+#else
+#define __noretpoline
+#endif
+
 /* These macros are used to mark some functions or 
  * initialized data (doesn't apply to uninitialized data)
  * as `initialization' functions. The kernel can take this
@@ -40,7 +47,7 @@
 
 /* These are for everybody (although not all archs will actually
discard it in modules) */
-#define __init __section(.init.text) __cold  __latent_entropy
+#define __init __section(.init.text) __cold  __latent_entropy 
__noretpoline
 #define __initdata __section(.init.data)
 #define __initconst__section(.init.rodata)
 #define __exitdata __section(.exit.data)

[tip:x86/pti] x86/cpuid: Fix up "virtual" IBRS/IBPB/STIBP feature bits on Intel

[tip-bot for David Woodhouse]

Commit-ID:  7fcae1118f5fd44a862aa5c3525248e35ee67c3b
Gitweb: https://git.kernel.org/tip/7fcae1118f5fd44a862aa5c3525248e35ee67c3b
Author: David Woodhouse 
AuthorDate: Tue, 30 Jan 2018 14:30:23 +
Committer:  Thomas Gleixner 
CommitDate: Tue, 30 Jan 2018 22:35:05 +0100

x86/cpuid: Fix up "virtual" IBRS/IBPB/STIBP feature bits on Intel

Despite the fact that all the other code there seems to be doing it, just
using set_cpu_cap() in early_intel_init() doesn't actually work.

For CPUs with PKU support, setup_pku() calls get_cpu_cap() after
c->c_init() has set those feature bits. That resets those bits back to what
was queried from the hardware.

Turning the bits off for bad microcode is easy to fix. That can just use
setup_clear_cpu_cap() to force them off for all CPUs.

I was less keen on forcing the feature bits *on* that way, just in case
of inconsistencies. I appreciate that the kernel is going to get this
utterly wrong if CPU features are not consistent, because it has already
applied alternatives by the time secondary CPUs are brought up.

But at least if setup_force_cpu_cap() isn't being used, we might have a
chance of *detecting* the lack of the corresponding bit and either
panicking or refusing to bring the offending CPU online.

So ensure that the appropriate feature bits are set within get_cpu_cap()
regardless of how many extra times it's called.

Fixes: 2961298e ("x86/cpufeatures: Clean up Spectre v2 related CPUID flags")
Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: karah...@amazon.de
Cc: pet...@infradead.org
Cc: b...@alien8.de
Link: 
https://lkml.kernel.org/r/1517322623-15261-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/cpu/common.c | 21 +
 arch/x86/kernel/cpu/intel.c  | 27 ---
 2 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c7c996a..dd09270 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -750,6 +750,26 @@ static void apply_forced_caps(struct cpuinfo_x86 *c)
}
 }
 
+static void init_speculation_control(struct cpuinfo_x86 *c)
+{
+   /*
+* The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
+* and they also have a different bit for STIBP support. Also,
+* a hypervisor might have set the individual AMD bits even on
+* Intel CPUs, for finer-grained selection of what's available.
+*
+* We use the AMD bits in 0x8000_0008 EBX as the generic hardware
+* features, which are visible in /proc/cpuinfo and used by the
+* kernel. So set those accordingly from the Intel bits.
+*/
+   if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
+   set_cpu_cap(c, X86_FEATURE_IBRS);
+   set_cpu_cap(c, X86_FEATURE_IBPB);
+   }
+   if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
+   set_cpu_cap(c, X86_FEATURE_STIBP);
+}
+
 void get_cpu_cap(struct cpuinfo_x86 *c)
 {
u32 eax, ebx, ecx, edx;
@@ -844,6 +864,7 @@ void get_cpu_cap(struct cpuinfo_x86 *c)
c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x800a);
 
init_scattered_cpuid_features(c);
+   init_speculation_control(c);
 
/*
 * Clear/Set all flags overridden by options, after probe.
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 6936d14..319bf98 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -175,28 +175,17 @@ static void early_init_intel(struct cpuinfo_x86 *c)
if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64))
c->microcode = intel_get_microcode_revision();
 
-   /*
-* The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
-* and they also have a different bit for STIBP support. Also,
-* a hypervisor might have set the individual AMD bits even on
-* Intel CPUs, for finer-grained selection of what's available.
-*/
-   if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
-   set_cpu_cap(c, X86_FEATURE_IBRS);
-   set_cpu_cap(c, X86_FEATURE_IBPB);
-   }
-   if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
-   set_cpu_cap(c, X86_FEATURE_STIBP);
-
/* Now if any of them are set, check the blacklist and clear the lot */
-   if ((cpu_has(c, X86_FEATURE_IBRS) || cpu_has(c, X86_FEATURE_IBPB) ||
+   if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) ||
+cpu_has(c, X86_FEATURE_INTEL_STIBP) ||
+cpu_has(c, X86_FEATURE_IBRS) || cpu_has(c, X86_FEATURE_IBPB) ||
 cpu_has(c, X86_FEATURE_STIBP)) && bad_spectre_microcode(c)) {
pr_warn("Intel Spectre v2 broken microcode detected; disabling 
Speculation Control\n");
-   clear_cpu_cap(c, X86_FEATURE_IBRS);
-   clear_cpu_cap(c, X86_FEATURE_IBPB);
-

[tip:x86/pti] x86/cpuid: Fix up "virtual" IBRS/IBPB/STIBP feature bits on Intel

[tip-bot for David Woodhouse]

Commit-ID:  7fcae1118f5fd44a862aa5c3525248e35ee67c3b
Gitweb: https://git.kernel.org/tip/7fcae1118f5fd44a862aa5c3525248e35ee67c3b
Author: David Woodhouse 
AuthorDate: Tue, 30 Jan 2018 14:30:23 +
Committer:  Thomas Gleixner 
CommitDate: Tue, 30 Jan 2018 22:35:05 +0100

x86/cpuid: Fix up "virtual" IBRS/IBPB/STIBP feature bits on Intel

Despite the fact that all the other code there seems to be doing it, just
using set_cpu_cap() in early_intel_init() doesn't actually work.

For CPUs with PKU support, setup_pku() calls get_cpu_cap() after
c->c_init() has set those feature bits. That resets those bits back to what
was queried from the hardware.

Turning the bits off for bad microcode is easy to fix. That can just use
setup_clear_cpu_cap() to force them off for all CPUs.

I was less keen on forcing the feature bits *on* that way, just in case
of inconsistencies. I appreciate that the kernel is going to get this
utterly wrong if CPU features are not consistent, because it has already
applied alternatives by the time secondary CPUs are brought up.

But at least if setup_force_cpu_cap() isn't being used, we might have a
chance of *detecting* the lack of the corresponding bit and either
panicking or refusing to bring the offending CPU online.

So ensure that the appropriate feature bits are set within get_cpu_cap()
regardless of how many extra times it's called.

Fixes: 2961298e ("x86/cpufeatures: Clean up Spectre v2 related CPUID flags")
Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: karah...@amazon.de
Cc: pet...@infradead.org
Cc: b...@alien8.de
Link: 
https://lkml.kernel.org/r/1517322623-15261-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/cpu/common.c | 21 +
 arch/x86/kernel/cpu/intel.c  | 27 ---
 2 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c7c996a..dd09270 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -750,6 +750,26 @@ static void apply_forced_caps(struct cpuinfo_x86 *c)
}
 }
 
+static void init_speculation_control(struct cpuinfo_x86 *c)
+{
+   /*
+* The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
+* and they also have a different bit for STIBP support. Also,
+* a hypervisor might have set the individual AMD bits even on
+* Intel CPUs, for finer-grained selection of what's available.
+*
+* We use the AMD bits in 0x8000_0008 EBX as the generic hardware
+* features, which are visible in /proc/cpuinfo and used by the
+* kernel. So set those accordingly from the Intel bits.
+*/
+   if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
+   set_cpu_cap(c, X86_FEATURE_IBRS);
+   set_cpu_cap(c, X86_FEATURE_IBPB);
+   }
+   if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
+   set_cpu_cap(c, X86_FEATURE_STIBP);
+}
+
 void get_cpu_cap(struct cpuinfo_x86 *c)
 {
u32 eax, ebx, ecx, edx;
@@ -844,6 +864,7 @@ void get_cpu_cap(struct cpuinfo_x86 *c)
c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x800a);
 
init_scattered_cpuid_features(c);
+   init_speculation_control(c);
 
/*
 * Clear/Set all flags overridden by options, after probe.
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 6936d14..319bf98 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -175,28 +175,17 @@ static void early_init_intel(struct cpuinfo_x86 *c)
if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64))
c->microcode = intel_get_microcode_revision();
 
-   /*
-* The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
-* and they also have a different bit for STIBP support. Also,
-* a hypervisor might have set the individual AMD bits even on
-* Intel CPUs, for finer-grained selection of what's available.
-*/
-   if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
-   set_cpu_cap(c, X86_FEATURE_IBRS);
-   set_cpu_cap(c, X86_FEATURE_IBPB);
-   }
-   if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
-   set_cpu_cap(c, X86_FEATURE_STIBP);
-
/* Now if any of them are set, check the blacklist and clear the lot */
-   if ((cpu_has(c, X86_FEATURE_IBRS) || cpu_has(c, X86_FEATURE_IBPB) ||
+   if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) ||
+cpu_has(c, X86_FEATURE_INTEL_STIBP) ||
+cpu_has(c, X86_FEATURE_IBRS) || cpu_has(c, X86_FEATURE_IBPB) ||
 cpu_has(c, X86_FEATURE_STIBP)) && bad_spectre_microcode(c)) {
pr_warn("Intel Spectre v2 broken microcode detected; disabling 
Speculation Control\n");
-   clear_cpu_cap(c, X86_FEATURE_IBRS);
-   clear_cpu_cap(c, X86_FEATURE_IBPB);
-   clear_cpu_cap(c, X86_FEATURE_STIBP);
-   clear_cpu_cap(c, 

[tip:x86/pti] x86/cpufeatures: Clean up Spectre v2 related CPUID flags

[tip-bot for David Woodhouse]

Commit-ID:  2961298efe1ea1b6fc0d7ee8b76018fa6c0bcef2
Gitweb: https://git.kernel.org/tip/2961298efe1ea1b6fc0d7ee8b76018fa6c0bcef2
Author: David Woodhouse 
AuthorDate: Sat, 27 Jan 2018 16:24:32 +
Committer:  Thomas Gleixner 
CommitDate: Sat, 27 Jan 2018 19:10:44 +0100

x86/cpufeatures: Clean up Spectre v2 related CPUID flags

We want to expose the hardware features simply in /proc/cpuinfo as "ibrs",
"ibpb" and "stibp". Since AMD has separate CPUID bits for those, use them
as the user-visible bits.

When the Intel SPEC_CTRL bit is set which indicates both IBRS and IBPB
capability, set those (AMD) bits accordingly. Likewise if the Intel STIBP
bit is set, set the AMD STIBP that's used for the generic hardware
capability.

Hide the rest from /proc/cpuinfo by putting "" in the comments. Including
RETPOLINE and RETPOLINE_AMD which shouldn't be visible there. There are
patches to make the sysfs vulnerabilities information non-readable by
non-root, and the same should apply to all information about which
mitigations are actually in use. Those *shouldn't* appear in /proc/cpuinfo.

The feature bit for whether IBPB is actually used, which is needed for
ALTERNATIVEs, is renamed to X86_FEATURE_USE_IBPB.

Originally-by: Borislav Petkov 
Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: a...@linux.intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1517070274-12128-2-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h   | 18 +-
 arch/x86/include/asm/nospec-branch.h |  2 +-
 arch/x86/kernel/cpu/bugs.c   |  7 +++
 arch/x86/kernel/cpu/intel.c  | 31 +--
 4 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 07934b2..73b5fff 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,14 +203,14 @@
 #define X86_FEATURE_PROC_FEEDBACK  ( 7*32+ 9) /* AMD ProcFeedbackInterface 
*/
 #define X86_FEATURE_SME( 7*32+10) /* AMD Secure Memory 
Encryption */
 #define X86_FEATURE_PTI( 7*32+11) /* Kernel Page Table 
Isolation enabled */
-#define X86_FEATURE_RETPOLINE  ( 7*32+12) /* Generic Retpoline 
mitigation for Spectre variant 2 */
-#define X86_FEATURE_RETPOLINE_AMD  ( 7*32+13) /* AMD Retpoline mitigation 
for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE  ( 7*32+12) /* "" Generic Retpoline 
mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD  ( 7*32+13) /* "" AMD Retpoline 
mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory 
Number */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
-#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
+#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* "" Fill RSB on context 
switches */
 
-#define X86_FEATURE_IBPB   ( 7*32+21) /* Indirect Branch 
Prediction Barrier enabled*/
+#define X86_FEATURE_USE_IBPB   ( 7*32+21) /* "" Indirect Branch 
Prediction Barrier enabled */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
@@ -271,9 +271,9 @@
 #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */
 #define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired 
Count */
 #define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP 
error pointers */
-#define X86_FEATURE_AMD_PRED_CMD   (13*32+12) /* Prediction Command MSR 
(AMD) */
-#define X86_FEATURE_AMD_SPEC_CTRL  (13*32+14) /* Speculation Control MSR 
only (AMD) */
-#define X86_FEATURE_AMD_STIBP  (13*32+15) /* Single Thread Indirect 
Branch Predictors (AMD) */
+#define X86_FEATURE_IBPB   (13*32+12) /* Indirect Branch 
Prediction Barrier */
+#define X86_FEATURE_IBRS   (13*32+14) /* Indirect Branch 
Restricted Speculation */
+#define X86_FEATURE_STIBP  (13*32+15) /* Single Thread Indirect 
Branch Predictors */
 
 /* Thermal and Power Management Leaf, CPUID level 0x0006 (EAX), word 14 */
 #define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */
@@ -325,8 +325,8 @@
 /* Intel-defined CPU features, CPUID level 0x0007:0 (EDX), word 18 */
 #define X86_FEATURE_AVX512_4VNNIW  (18*32+ 2) /* AVX-512 Neural Network 
Instructions */
 #define X86_FEATURE_AVX512_4FMAPS  (18*32+ 3) /* AVX-512 Multiply 
Accumulation Single precision */
-#define 

[tip:x86/pti] x86/cpufeatures: Clean up Spectre v2 related CPUID flags

[tip-bot for David Woodhouse]

Commit-ID:  2961298efe1ea1b6fc0d7ee8b76018fa6c0bcef2
Gitweb: https://git.kernel.org/tip/2961298efe1ea1b6fc0d7ee8b76018fa6c0bcef2
Author: David Woodhouse 
AuthorDate: Sat, 27 Jan 2018 16:24:32 +
Committer:  Thomas Gleixner 
CommitDate: Sat, 27 Jan 2018 19:10:44 +0100

x86/cpufeatures: Clean up Spectre v2 related CPUID flags

We want to expose the hardware features simply in /proc/cpuinfo as "ibrs",
"ibpb" and "stibp". Since AMD has separate CPUID bits for those, use them
as the user-visible bits.

When the Intel SPEC_CTRL bit is set which indicates both IBRS and IBPB
capability, set those (AMD) bits accordingly. Likewise if the Intel STIBP
bit is set, set the AMD STIBP that's used for the generic hardware
capability.

Hide the rest from /proc/cpuinfo by putting "" in the comments. Including
RETPOLINE and RETPOLINE_AMD which shouldn't be visible there. There are
patches to make the sysfs vulnerabilities information non-readable by
non-root, and the same should apply to all information about which
mitigations are actually in use. Those *shouldn't* appear in /proc/cpuinfo.

The feature bit for whether IBPB is actually used, which is needed for
ALTERNATIVEs, is renamed to X86_FEATURE_USE_IBPB.

Originally-by: Borislav Petkov 
Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: a...@linux.intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1517070274-12128-2-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h   | 18 +-
 arch/x86/include/asm/nospec-branch.h |  2 +-
 arch/x86/kernel/cpu/bugs.c   |  7 +++
 arch/x86/kernel/cpu/intel.c  | 31 +--
 4 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 07934b2..73b5fff 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,14 +203,14 @@
 #define X86_FEATURE_PROC_FEEDBACK  ( 7*32+ 9) /* AMD ProcFeedbackInterface 
*/
 #define X86_FEATURE_SME( 7*32+10) /* AMD Secure Memory 
Encryption */
 #define X86_FEATURE_PTI( 7*32+11) /* Kernel Page Table 
Isolation enabled */
-#define X86_FEATURE_RETPOLINE  ( 7*32+12) /* Generic Retpoline 
mitigation for Spectre variant 2 */
-#define X86_FEATURE_RETPOLINE_AMD  ( 7*32+13) /* AMD Retpoline mitigation 
for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE  ( 7*32+12) /* "" Generic Retpoline 
mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD  ( 7*32+13) /* "" AMD Retpoline 
mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory 
Number */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
-#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
+#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* "" Fill RSB on context 
switches */
 
-#define X86_FEATURE_IBPB   ( 7*32+21) /* Indirect Branch 
Prediction Barrier enabled*/
+#define X86_FEATURE_USE_IBPB   ( 7*32+21) /* "" Indirect Branch 
Prediction Barrier enabled */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
@@ -271,9 +271,9 @@
 #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */
 #define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired 
Count */
 #define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP 
error pointers */
-#define X86_FEATURE_AMD_PRED_CMD   (13*32+12) /* Prediction Command MSR 
(AMD) */
-#define X86_FEATURE_AMD_SPEC_CTRL  (13*32+14) /* Speculation Control MSR 
only (AMD) */
-#define X86_FEATURE_AMD_STIBP  (13*32+15) /* Single Thread Indirect 
Branch Predictors (AMD) */
+#define X86_FEATURE_IBPB   (13*32+12) /* Indirect Branch 
Prediction Barrier */
+#define X86_FEATURE_IBRS   (13*32+14) /* Indirect Branch 
Restricted Speculation */
+#define X86_FEATURE_STIBP  (13*32+15) /* Single Thread Indirect 
Branch Predictors */
 
 /* Thermal and Power Management Leaf, CPUID level 0x0006 (EAX), word 14 */
 #define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */
@@ -325,8 +325,8 @@
 /* Intel-defined CPU features, CPUID level 0x0007:0 (EDX), word 18 */
 #define X86_FEATURE_AVX512_4VNNIW  (18*32+ 2) /* AVX-512 Neural Network 
Instructions */
 #define X86_FEATURE_AVX512_4FMAPS  (18*32+ 3) /* AVX-512 Multiply 
Accumulation Single precision */
-#define X86_FEATURE_SPEC_CTRL  (18*32+26) /* Speculation Control (IBRS 
+ IBPB) */
-#define 

[tip:x86/pti] x86/pti: Do not enable PTI on CPUs which are not vulnerable to Meltdown

[tip-bot for David Woodhouse]

Commit-ID:  fec9434a12f38d3aeafeb75711b71d8a1fdef621
Gitweb: https://git.kernel.org/tip/fec9434a12f38d3aeafeb75711b71d8a1fdef621
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:13 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:18 +0100

x86/pti: Do not enable PTI on CPUs which are not vulnerable to Meltdown

Also, for CPUs which don't speculate at all, don't report that they're
vulnerable to the Spectre variants either.

Leave the cpu_no_meltdown[] match table with just X86_VENDOR_AMD in it
for now, even though that could be done with a simple comparison, on the
assumption that we'll have more to add.

Based on suggestions from Dave Hansen and Alan Cox.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Reviewed-by: Borislav Petkov 
Acked-by: Dave Hansen 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-6-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/cpu/common.c | 48 +++-
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index e5d66e9..970ee06 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -47,6 +47,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #ifdef CONFIG_X86_LOCAL_APIC
 #include 
@@ -853,6 +855,41 @@ static void identify_cpu_without_cpuid(struct cpuinfo_x86 
*c)
 #endif
 }
 
+static const __initdata struct x86_cpu_id cpu_no_speculation[] = {
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CEDARVIEW,   X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CLOVERVIEW,  X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_LINCROFT,X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PENWELL, X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PINEVIEW,X86_FEATURE_ANY 
},
+   { X86_VENDOR_CENTAUR,   5 },
+   { X86_VENDOR_INTEL, 5 },
+   { X86_VENDOR_NSC,   5 },
+   { X86_VENDOR_ANY,   4 },
+   {}
+};
+
+static const __initdata struct x86_cpu_id cpu_no_meltdown[] = {
+   { X86_VENDOR_AMD },
+   {}
+};
+
+static bool __init cpu_vulnerable_to_meltdown(struct cpuinfo_x86 *c)
+{
+   u64 ia32_cap = 0;
+
+   if (x86_match_cpu(cpu_no_meltdown))
+   return false;
+
+   if (cpu_has(c, X86_FEATURE_ARCH_CAPABILITIES))
+   rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
+
+   /* Rogue Data Cache Load? No! */
+   if (ia32_cap & ARCH_CAP_RDCL_NO)
+   return false;
+
+   return true;
+}
+
 /*
  * Do minimum CPU detection early.
  * Fields really needed: vendor, cpuid_level, family, model, mask,
@@ -900,11 +937,12 @@ static void __init early_identify_cpu(struct cpuinfo_x86 
*c)
 
setup_force_cpu_cap(X86_FEATURE_ALWAYS);
 
-   if (c->x86_vendor != X86_VENDOR_AMD)
-   setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
-
-   setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
-   setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+   if (!x86_match_cpu(cpu_no_speculation)) {
+   if (cpu_vulnerable_to_meltdown(c))
+   setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
+   setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
+   setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+   }
 
fpu__init_system(c);
 

[tip:x86/pti] x86/pti: Do not enable PTI on CPUs which are not vulnerable to Meltdown

[tip-bot for David Woodhouse]

Commit-ID:  fec9434a12f38d3aeafeb75711b71d8a1fdef621
Gitweb: https://git.kernel.org/tip/fec9434a12f38d3aeafeb75711b71d8a1fdef621
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:13 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:18 +0100

x86/pti: Do not enable PTI on CPUs which are not vulnerable to Meltdown

Also, for CPUs which don't speculate at all, don't report that they're
vulnerable to the Spectre variants either.

Leave the cpu_no_meltdown[] match table with just X86_VENDOR_AMD in it
for now, even though that could be done with a simple comparison, on the
assumption that we'll have more to add.

Based on suggestions from Dave Hansen and Alan Cox.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Reviewed-by: Borislav Petkov 
Acked-by: Dave Hansen 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-6-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/cpu/common.c | 48 +++-
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index e5d66e9..970ee06 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -47,6 +47,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #ifdef CONFIG_X86_LOCAL_APIC
 #include 
@@ -853,6 +855,41 @@ static void identify_cpu_without_cpuid(struct cpuinfo_x86 
*c)
 #endif
 }
 
+static const __initdata struct x86_cpu_id cpu_no_speculation[] = {
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CEDARVIEW,   X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CLOVERVIEW,  X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_LINCROFT,X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PENWELL, X86_FEATURE_ANY 
},
+   { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PINEVIEW,X86_FEATURE_ANY 
},
+   { X86_VENDOR_CENTAUR,   5 },
+   { X86_VENDOR_INTEL, 5 },
+   { X86_VENDOR_NSC,   5 },
+   { X86_VENDOR_ANY,   4 },
+   {}
+};
+
+static const __initdata struct x86_cpu_id cpu_no_meltdown[] = {
+   { X86_VENDOR_AMD },
+   {}
+};
+
+static bool __init cpu_vulnerable_to_meltdown(struct cpuinfo_x86 *c)
+{
+   u64 ia32_cap = 0;
+
+   if (x86_match_cpu(cpu_no_meltdown))
+   return false;
+
+   if (cpu_has(c, X86_FEATURE_ARCH_CAPABILITIES))
+   rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
+
+   /* Rogue Data Cache Load? No! */
+   if (ia32_cap & ARCH_CAP_RDCL_NO)
+   return false;
+
+   return true;
+}
+
 /*
  * Do minimum CPU detection early.
  * Fields really needed: vendor, cpuid_level, family, model, mask,
@@ -900,11 +937,12 @@ static void __init early_identify_cpu(struct cpuinfo_x86 
*c)
 
setup_force_cpu_cap(X86_FEATURE_ALWAYS);
 
-   if (c->x86_vendor != X86_VENDOR_AMD)
-   setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
-
-   setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
-   setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+   if (!x86_match_cpu(cpu_no_speculation)) {
+   if (cpu_vulnerable_to_meltdown(c))
+   setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
+   setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
+   setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+   }
 
fpu__init_system(c);
 

[tip:x86/pti] x86/cpufeatures: Add Intel feature bits for Speculation Control

[tip-bot for David Woodhouse]

Commit-ID:  fc67dd70adb711a45d2ef34e12d1a8be75edde61
Gitweb: https://git.kernel.org/tip/fc67dd70adb711a45d2ef34e12d1a8be75edde61
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:10 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:16 +0100

x86/cpufeatures: Add Intel feature bits for Speculation Control

Add three feature bits exposed by new microcode on Intel CPUs for
speculation control.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Reviewed-by: Borislav Petkov 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-3-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 7b25cf3..0a51070 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -320,6 +320,9 @@
 /* Intel-defined CPU features, CPUID level 0x0007:0 (EDX), word 18 */
 #define X86_FEATURE_AVX512_4VNNIW  (18*32+ 2) /* AVX-512 Neural Network 
Instructions */
 #define X86_FEATURE_AVX512_4FMAPS  (18*32+ 3) /* AVX-512 Multiply 
Accumulation Single precision */
+#define X86_FEATURE_SPEC_CTRL  (18*32+26) /* Speculation Control (IBRS 
+ IBPB) */
+#define X86_FEATURE_STIBP  (18*32+27) /* Single Thread Indirect 
Branch Predictors */
+#define X86_FEATURE_ARCH_CAPABILITIES  (18*32+29) /* IA32_ARCH_CAPABILITIES 
MSR (Intel) */
 
 /*
  * BUG word(s)

[tip:x86/pti] x86/cpufeatures: Add Intel feature bits for Speculation Control

[tip-bot for David Woodhouse]

Commit-ID:  fc67dd70adb711a45d2ef34e12d1a8be75edde61
Gitweb: https://git.kernel.org/tip/fc67dd70adb711a45d2ef34e12d1a8be75edde61
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:10 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:16 +0100

x86/cpufeatures: Add Intel feature bits for Speculation Control

Add three feature bits exposed by new microcode on Intel CPUs for
speculation control.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Reviewed-by: Borislav Petkov 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-3-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 7b25cf3..0a51070 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -320,6 +320,9 @@
 /* Intel-defined CPU features, CPUID level 0x0007:0 (EDX), word 18 */
 #define X86_FEATURE_AVX512_4VNNIW  (18*32+ 2) /* AVX-512 Neural Network 
Instructions */
 #define X86_FEATURE_AVX512_4FMAPS  (18*32+ 3) /* AVX-512 Multiply 
Accumulation Single precision */
+#define X86_FEATURE_SPEC_CTRL  (18*32+26) /* Speculation Control (IBRS 
+ IBPB) */
+#define X86_FEATURE_STIBP  (18*32+27) /* Single Thread Indirect 
Branch Predictors */
+#define X86_FEATURE_ARCH_CAPABILITIES  (18*32+29) /* IA32_ARCH_CAPABILITIES 
MSR (Intel) */
 
 /*
  * BUG word(s)

[tip:x86/pti] x86/cpufeatures: Add CPUID_7_EDX CPUID leaf

[tip-bot for David Woodhouse]

Commit-ID:  95ca0ee8636059ea2800dfbac9ecac6212d6b38f
Gitweb: https://git.kernel.org/tip/95ca0ee8636059ea2800dfbac9ecac6212d6b38f
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:09 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:16 +0100

x86/cpufeatures: Add CPUID_7_EDX CPUID leaf

This is a pure feature bits leaf. There are two AVX512 feature bits in it
already which were handled as scattered bits, and three more from this leaf
are going to be added for speculation control features.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Reviewed-by: Borislav Petkov 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-2-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeature.h| 7 +--
 arch/x86/include/asm/cpufeatures.h   | 8 +---
 arch/x86/include/asm/disabled-features.h | 3 ++-
 arch/x86/include/asm/required-features.h | 3 ++-
 arch/x86/kernel/cpu/common.c | 1 +
 arch/x86/kernel/cpu/scattered.c  | 2 --
 6 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/arch/x86/include/asm/cpufeature.h 
b/arch/x86/include/asm/cpufeature.h
index ea9a7dd..70eddb3 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -29,6 +29,7 @@ enum cpuid_leafs
CPUID_8000_000A_EDX,
CPUID_7_ECX,
CPUID_8000_0007_EBX,
+   CPUID_7_EDX,
 };
 
 #ifdef CONFIG_X86_FEATURE_NAMES
@@ -79,8 +80,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||\
+  CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||\
   REQUIRED_MASK_CHECK||\
-  BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+  BUILD_BUG_ON_ZERO(NCAPINTS != 19))
 
 #define DISABLED_MASK_BIT_SET(feature_bit) \
 ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||\
@@ -101,8 +103,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||\
+  CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||\
   DISABLED_MASK_CHECK||\
-  BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+  BUILD_BUG_ON_ZERO(NCAPINTS != 19))
 
 #define cpu_has(c, bit)
\
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :  \
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 25b9375..7b25cf3 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -13,7 +13,7 @@
 /*
  * Defines x86 CPU feature bits
  */
-#define NCAPINTS   18 /* N 32-bit words worth of 
info */
+#define NCAPINTS   19 /* N 32-bit words worth of 
info */
 #define NBUGINTS   1  /* N 32-bit bug flags */
 
 /*
@@ -206,8 +206,6 @@
 #define X86_FEATURE_RETPOLINE  ( 7*32+12) /* Generic Retpoline 
mitigation for Spectre variant 2 */
 #define X86_FEATURE_RETPOLINE_AMD  ( 7*32+13) /* AMD Retpoline mitigation 
for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory 
Number */
-#define X86_FEATURE_AVX512_4VNNIW  ( 7*32+16) /* AVX-512 Neural Network 
Instructions */
-#define X86_FEATURE_AVX512_4FMAPS  ( 7*32+17) /* AVX-512 Multiply 
Accumulation Single precision */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
 #define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
@@ -319,6 +317,10 @@
 #define X86_FEATURE_SUCCOR (17*32+ 1) /* Uncorrectable error 
containment and recovery */
 #define X86_FEATURE_SMCA   (17*32+ 3) /* Scalable MCA */
 
+/* Intel-defined CPU features, CPUID level 0x0007:0 (EDX), word 18 */
+#define X86_FEATURE_AVX512_4VNNIW  (18*32+ 2) /* AVX-512 Neural Network 
Instructions */
+#define X86_FEATURE_AVX512_4FMAPS  (18*32+ 3) /* AVX-512 Multiply 
Accumulation Single precision */
+
 /*
  * BUG 

[tip:x86/pti] x86/cpufeatures: Add CPUID_7_EDX CPUID leaf

[tip-bot for David Woodhouse]

Commit-ID:  95ca0ee8636059ea2800dfbac9ecac6212d6b38f
Gitweb: https://git.kernel.org/tip/95ca0ee8636059ea2800dfbac9ecac6212d6b38f
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:09 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:16 +0100

x86/cpufeatures: Add CPUID_7_EDX CPUID leaf

This is a pure feature bits leaf. There are two AVX512 feature bits in it
already which were handled as scattered bits, and three more from this leaf
are going to be added for speculation control features.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Reviewed-by: Borislav Petkov 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-2-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeature.h| 7 +--
 arch/x86/include/asm/cpufeatures.h   | 8 +---
 arch/x86/include/asm/disabled-features.h | 3 ++-
 arch/x86/include/asm/required-features.h | 3 ++-
 arch/x86/kernel/cpu/common.c | 1 +
 arch/x86/kernel/cpu/scattered.c  | 2 --
 6 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/arch/x86/include/asm/cpufeature.h 
b/arch/x86/include/asm/cpufeature.h
index ea9a7dd..70eddb3 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -29,6 +29,7 @@ enum cpuid_leafs
CPUID_8000_000A_EDX,
CPUID_7_ECX,
CPUID_8000_0007_EBX,
+   CPUID_7_EDX,
 };
 
 #ifdef CONFIG_X86_FEATURE_NAMES
@@ -79,8 +80,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||\
+  CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||\
   REQUIRED_MASK_CHECK||\
-  BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+  BUILD_BUG_ON_ZERO(NCAPINTS != 19))
 
 #define DISABLED_MASK_BIT_SET(feature_bit) \
 ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||\
@@ -101,8 +103,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||\
   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||\
+  CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||\
   DISABLED_MASK_CHECK||\
-  BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+  BUILD_BUG_ON_ZERO(NCAPINTS != 19))
 
 #define cpu_has(c, bit)
\
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :  \
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 25b9375..7b25cf3 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -13,7 +13,7 @@
 /*
  * Defines x86 CPU feature bits
  */
-#define NCAPINTS   18 /* N 32-bit words worth of 
info */
+#define NCAPINTS   19 /* N 32-bit words worth of 
info */
 #define NBUGINTS   1  /* N 32-bit bug flags */
 
 /*
@@ -206,8 +206,6 @@
 #define X86_FEATURE_RETPOLINE  ( 7*32+12) /* Generic Retpoline 
mitigation for Spectre variant 2 */
 #define X86_FEATURE_RETPOLINE_AMD  ( 7*32+13) /* AMD Retpoline mitigation 
for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory 
Number */
-#define X86_FEATURE_AVX512_4VNNIW  ( 7*32+16) /* AVX-512 Neural Network 
Instructions */
-#define X86_FEATURE_AVX512_4FMAPS  ( 7*32+17) /* AVX-512 Multiply 
Accumulation Single precision */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
 #define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
@@ -319,6 +317,10 @@
 #define X86_FEATURE_SUCCOR (17*32+ 1) /* Uncorrectable error 
containment and recovery */
 #define X86_FEATURE_SMCA   (17*32+ 3) /* Scalable MCA */
 
+/* Intel-defined CPU features, CPUID level 0x0007:0 (EDX), word 18 */
+#define X86_FEATURE_AVX512_4VNNIW  (18*32+ 2) /* AVX-512 Neural Network 
Instructions */
+#define X86_FEATURE_AVX512_4FMAPS  (18*32+ 3) /* AVX-512 Multiply 
Accumulation Single precision */
+
 /*
  * BUG word(s)
  */
diff --git a/arch/x86/include/asm/disabled-features.h 
b/arch/x86/include/asm/disabled-features.h
index 

[tip:x86/pti] x86/cpufeature: Blacklist SPEC_CTRL/PRED_CMD on early Spectre v2 microcodes

[tip-bot for David Woodhouse]

Commit-ID:  a5b2966364538a0e68c9fa29bc0a3a1651799035
Gitweb: https://git.kernel.org/tip/a5b2966364538a0e68c9fa29bc0a3a1651799035
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:14 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:18 +0100

x86/cpufeature: Blacklist SPEC_CTRL/PRED_CMD on early Spectre v2 microcodes

This doesn't refuse to load the affected microcodes; it just refuses to
use the Spectre v2 mitigation features if they're detected, by clearing
the appropriate feature bits.

The AMD CPUID bits are handled here too, because hypervisors *may* have
been exposing those bits even on Intel chips, for fine-grained control
of what's available.

It is non-trivial to use x86_match_cpu() for this table because that
doesn't handle steppings. And the approach taken in commit bd9240a18
almost made me lose my lunch.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-7-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/cpu/intel.c | 66 +
 1 file changed, 66 insertions(+)

diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index b720dac..5faa487 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -102,6 +102,59 @@ static void probe_xeon_phi_r3mwait(struct cpuinfo_x86 *c)
ELF_HWCAP2 |= HWCAP2_RING3MWAIT;
 }
 
+/*
+ * Early microcode releases for the Spectre v2 mitigation were broken.
+ * Information taken from;
+ * - 
https://newsroom.intel.com/wp-content/uploads/sites/11/2018/01/microcode-update-guidance.pdf
+ * - https://kb.vmware.com/s/article/52345
+ * - Microcode revisions observed in the wild
+ * - Release note from 20180108 microcode release
+ */
+struct sku_microcode {
+   u8 model;
+   u8 stepping;
+   u32 microcode;
+};
+static const struct sku_microcode spectre_bad_microcodes[] = {
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0B,   0x84 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0A,   0x84 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x09,   0x84 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x0A,   0x84 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x84 },
+   { INTEL_FAM6_SKYLAKE_X, 0x03,   0x0100013e },
+   { INTEL_FAM6_SKYLAKE_X, 0x04,   0x023c },
+   { INTEL_FAM6_SKYLAKE_MOBILE,0x03,   0xc2 },
+   { INTEL_FAM6_SKYLAKE_DESKTOP,   0x03,   0xc2 },
+   { INTEL_FAM6_BROADWELL_CORE,0x04,   0x28 },
+   { INTEL_FAM6_BROADWELL_GT3E,0x01,   0x1b },
+   { INTEL_FAM6_BROADWELL_XEON_D,  0x02,   0x14 },
+   { INTEL_FAM6_BROADWELL_XEON_D,  0x03,   0x0711 },
+   { INTEL_FAM6_BROADWELL_X,   0x01,   0x0b25 },
+   { INTEL_FAM6_HASWELL_ULT,   0x01,   0x21 },
+   { INTEL_FAM6_HASWELL_GT3E,  0x01,   0x18 },
+   { INTEL_FAM6_HASWELL_CORE,  0x03,   0x23 },
+   { INTEL_FAM6_HASWELL_X, 0x02,   0x3b },
+   { INTEL_FAM6_HASWELL_X, 0x04,   0x10 },
+   { INTEL_FAM6_IVYBRIDGE_X,   0x04,   0x42a },
+   /* Updated in the 20180108 release; blacklist until we know otherwise */
+   { INTEL_FAM6_ATOM_GEMINI_LAKE,  0x01,   0x22 },
+   /* Observed in the wild */
+   { INTEL_FAM6_SANDYBRIDGE_X, 0x06,   0x61b },
+   { INTEL_FAM6_SANDYBRIDGE_X, 0x07,   0x712 },
+};
+
+static bool bad_spectre_microcode(struct cpuinfo_x86 *c)
+{
+   int i;
+
+   for (i = 0; i < ARRAY_SIZE(spectre_bad_microcodes); i++) {
+   if (c->x86_model == spectre_bad_microcodes[i].model &&
+   c->x86_mask == spectre_bad_microcodes[i].stepping)
+   return (c->microcode <= 
spectre_bad_microcodes[i].microcode);
+   }
+   return false;
+}
+
 static void early_init_intel(struct cpuinfo_x86 *c)
 {
u64 misc_enable;
@@ -122,6 +175,19 @@ static void early_init_intel(struct cpuinfo_x86 *c)
if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64))
c->microcode = intel_get_microcode_revision();
 
+   if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) ||
+cpu_has(c, X86_FEATURE_STIBP) ||
+cpu_has(c, X86_FEATURE_AMD_SPEC_CTRL) ||
+cpu_has(c, X86_FEATURE_AMD_PRED_CMD) ||
+cpu_has(c, X86_FEATURE_AMD_STIBP)) && bad_spectre_microcode(c)) {
+   pr_warn("Intel Spectre v2 broken microcode detected; disabling 
SPEC_CTRL\n");
+   clear_cpu_cap(c, X86_FEATURE_SPEC_CTRL);
+   clear_cpu_cap(c, 

[tip:x86/pti] x86/cpufeatures: Add AMD feature bits for Speculation Control

[tip-bot for David Woodhouse]

Commit-ID:  5d10cbc91d9eb5537998b65608441b592eec65e7
Gitweb: https://git.kernel.org/tip/5d10cbc91d9eb5537998b65608441b592eec65e7
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:11 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:17 +0100

x86/cpufeatures: Add AMD feature bits for Speculation Control

AMD exposes the PRED_CMD/SPEC_CTRL MSRs slightly differently to Intel.
See http://lkml.kernel.org/r/2b3e25cc-286d-8bd0-aeaf-9ac4aae39...@amd.com

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Cc: Tom Lendacky 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-4-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 0a51070..ae3212f 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -269,6 +269,9 @@
 #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */
 #define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired 
Count */
 #define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP 
error pointers */
+#define X86_FEATURE_AMD_PRED_CMD   (13*32+12) /* Prediction Command MSR 
(AMD) */
+#define X86_FEATURE_AMD_SPEC_CTRL  (13*32+14) /* Speculation Control MSR 
only (AMD) */
+#define X86_FEATURE_AMD_STIBP  (13*32+15) /* Single Thread Indirect 
Branch Predictors (AMD) */
 
 /* Thermal and Power Management Leaf, CPUID level 0x0006 (EAX), word 14 */
 #define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */

[tip:x86/pti] x86/cpufeature: Blacklist SPEC_CTRL/PRED_CMD on early Spectre v2 microcodes

[tip-bot for David Woodhouse]

Commit-ID:  a5b2966364538a0e68c9fa29bc0a3a1651799035
Gitweb: https://git.kernel.org/tip/a5b2966364538a0e68c9fa29bc0a3a1651799035
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:14 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:18 +0100

x86/cpufeature: Blacklist SPEC_CTRL/PRED_CMD on early Spectre v2 microcodes

This doesn't refuse to load the affected microcodes; it just refuses to
use the Spectre v2 mitigation features if they're detected, by clearing
the appropriate feature bits.

The AMD CPUID bits are handled here too, because hypervisors *may* have
been exposing those bits even on Intel chips, for fine-grained control
of what's available.

It is non-trivial to use x86_match_cpu() for this table because that
doesn't handle steppings. And the approach taken in commit bd9240a18
almost made me lose my lunch.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-7-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/cpu/intel.c | 66 +
 1 file changed, 66 insertions(+)

diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index b720dac..5faa487 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -102,6 +102,59 @@ static void probe_xeon_phi_r3mwait(struct cpuinfo_x86 *c)
ELF_HWCAP2 |= HWCAP2_RING3MWAIT;
 }
 
+/*
+ * Early microcode releases for the Spectre v2 mitigation were broken.
+ * Information taken from;
+ * - 
https://newsroom.intel.com/wp-content/uploads/sites/11/2018/01/microcode-update-guidance.pdf
+ * - https://kb.vmware.com/s/article/52345
+ * - Microcode revisions observed in the wild
+ * - Release note from 20180108 microcode release
+ */
+struct sku_microcode {
+   u8 model;
+   u8 stepping;
+   u32 microcode;
+};
+static const struct sku_microcode spectre_bad_microcodes[] = {
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0B,   0x84 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x0A,   0x84 },
+   { INTEL_FAM6_KABYLAKE_DESKTOP,  0x09,   0x84 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x0A,   0x84 },
+   { INTEL_FAM6_KABYLAKE_MOBILE,   0x09,   0x84 },
+   { INTEL_FAM6_SKYLAKE_X, 0x03,   0x0100013e },
+   { INTEL_FAM6_SKYLAKE_X, 0x04,   0x023c },
+   { INTEL_FAM6_SKYLAKE_MOBILE,0x03,   0xc2 },
+   { INTEL_FAM6_SKYLAKE_DESKTOP,   0x03,   0xc2 },
+   { INTEL_FAM6_BROADWELL_CORE,0x04,   0x28 },
+   { INTEL_FAM6_BROADWELL_GT3E,0x01,   0x1b },
+   { INTEL_FAM6_BROADWELL_XEON_D,  0x02,   0x14 },
+   { INTEL_FAM6_BROADWELL_XEON_D,  0x03,   0x0711 },
+   { INTEL_FAM6_BROADWELL_X,   0x01,   0x0b25 },
+   { INTEL_FAM6_HASWELL_ULT,   0x01,   0x21 },
+   { INTEL_FAM6_HASWELL_GT3E,  0x01,   0x18 },
+   { INTEL_FAM6_HASWELL_CORE,  0x03,   0x23 },
+   { INTEL_FAM6_HASWELL_X, 0x02,   0x3b },
+   { INTEL_FAM6_HASWELL_X, 0x04,   0x10 },
+   { INTEL_FAM6_IVYBRIDGE_X,   0x04,   0x42a },
+   /* Updated in the 20180108 release; blacklist until we know otherwise */
+   { INTEL_FAM6_ATOM_GEMINI_LAKE,  0x01,   0x22 },
+   /* Observed in the wild */
+   { INTEL_FAM6_SANDYBRIDGE_X, 0x06,   0x61b },
+   { INTEL_FAM6_SANDYBRIDGE_X, 0x07,   0x712 },
+};
+
+static bool bad_spectre_microcode(struct cpuinfo_x86 *c)
+{
+   int i;
+
+   for (i = 0; i < ARRAY_SIZE(spectre_bad_microcodes); i++) {
+   if (c->x86_model == spectre_bad_microcodes[i].model &&
+   c->x86_mask == spectre_bad_microcodes[i].stepping)
+   return (c->microcode <= 
spectre_bad_microcodes[i].microcode);
+   }
+   return false;
+}
+
 static void early_init_intel(struct cpuinfo_x86 *c)
 {
u64 misc_enable;
@@ -122,6 +175,19 @@ static void early_init_intel(struct cpuinfo_x86 *c)
if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64))
c->microcode = intel_get_microcode_revision();
 
+   if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) ||
+cpu_has(c, X86_FEATURE_STIBP) ||
+cpu_has(c, X86_FEATURE_AMD_SPEC_CTRL) ||
+cpu_has(c, X86_FEATURE_AMD_PRED_CMD) ||
+cpu_has(c, X86_FEATURE_AMD_STIBP)) && bad_spectre_microcode(c)) {
+   pr_warn("Intel Spectre v2 broken microcode detected; disabling 
SPEC_CTRL\n");
+   clear_cpu_cap(c, X86_FEATURE_SPEC_CTRL);
+   clear_cpu_cap(c, X86_FEATURE_STIBP);
+   clear_cpu_cap(c, X86_FEATURE_AMD_SPEC_CTRL);
+   

[tip:x86/pti] x86/cpufeatures: Add AMD feature bits for Speculation Control

[tip-bot for David Woodhouse]

Commit-ID:  5d10cbc91d9eb5537998b65608441b592eec65e7
Gitweb: https://git.kernel.org/tip/5d10cbc91d9eb5537998b65608441b592eec65e7
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:11 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:17 +0100

x86/cpufeatures: Add AMD feature bits for Speculation Control

AMD exposes the PRED_CMD/SPEC_CTRL MSRs slightly differently to Intel.
See http://lkml.kernel.org/r/2b3e25cc-286d-8bd0-aeaf-9ac4aae39...@amd.com

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Cc: Tom Lendacky 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-4-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 0a51070..ae3212f 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -269,6 +269,9 @@
 #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */
 #define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired 
Count */
 #define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP 
error pointers */
+#define X86_FEATURE_AMD_PRED_CMD   (13*32+12) /* Prediction Command MSR 
(AMD) */
+#define X86_FEATURE_AMD_SPEC_CTRL  (13*32+14) /* Speculation Control MSR 
only (AMD) */
+#define X86_FEATURE_AMD_STIBP  (13*32+15) /* Single Thread Indirect 
Branch Predictors (AMD) */
 
 /* Thermal and Power Management Leaf, CPUID level 0x0006 (EAX), word 14 */
 #define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */

[tip:x86/pti] x86/speculation: Add basic IBPB (Indirect Branch Prediction Barrier) support

[tip-bot for David Woodhouse]

Commit-ID:  20ffa1caecca4db8f79fe665acdeaa5af815a24d
Gitweb: https://git.kernel.org/tip/20ffa1caecca4db8f79fe665acdeaa5af815a24d
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:15 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:18 +0100

x86/speculation: Add basic IBPB (Indirect Branch Prediction Barrier) support

Expose indirect_branch_prediction_barrier() for use in subsequent patches.

[ tglx: Add IBPB status to spectre_v2 sysfs file ]

Co-developed-by: KarimAllah Ahmed 
Signed-off-by: KarimAllah Ahmed 
Signed-off-by: David Woodhouse 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-8-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h   |  2 ++
 arch/x86/include/asm/nospec-branch.h | 13 +
 arch/x86/kernel/cpu/bugs.c   | 10 +-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index ae3212f..07934b2 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -210,6 +210,8 @@
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
 #define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
 
+#define X86_FEATURE_IBPB   ( 7*32+21) /* Indirect Branch 
Prediction Barrier enabled*/
+
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
 #define X86_FEATURE_VNMI   ( 8*32+ 1) /* Intel Virtual NMI */
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 4ad4108..34e384c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -218,5 +218,18 @@ static inline void vmexit_fill_RSB(void)
 #endif
 }
 
+static inline void indirect_branch_prediction_barrier(void)
+{
+   asm volatile(ALTERNATIVE("",
+"movl %[msr], %%ecx\n\t"
+"movl %[val], %%eax\n\t"
+"movl $0, %%edx\n\t"
+"wrmsr",
+X86_FEATURE_IBPB)
+: : [msr] "i" (MSR_IA32_PRED_CMD),
+[val] "i" (PRED_CMD_IBPB)
+: "eax", "ecx", "edx", "memory");
+}
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 4a39d7b..bac7a35 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -263,6 +263,13 @@ retpoline_auto:
setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
pr_info("Filling RSB on context switch\n");
}
+
+   /* Initialize Indirect Branch Prediction Barrier if supported */
+   if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) ||
+   boot_cpu_has(X86_FEATURE_AMD_PRED_CMD)) {
+   setup_force_cpu_cap(X86_FEATURE_IBPB);
+   pr_info("Enabling Indirect Branch Prediction Barrier\n");
+   }
 }
 
 #undef pr_fmt
@@ -292,7 +299,8 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
return sprintf(buf, "Not affected\n");
 
-   return sprintf(buf, "%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+   return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+  boot_cpu_has(X86_FEATURE_IBPB) ? ", IPBP" : "",
   spectre_v2_bad_module ? " - vulnerable module loaded" : 
"");
 }
 #endif

[tip:x86/pti] x86/speculation: Add basic IBPB (Indirect Branch Prediction Barrier) support

[tip-bot for David Woodhouse]

Commit-ID:  20ffa1caecca4db8f79fe665acdeaa5af815a24d
Gitweb: https://git.kernel.org/tip/20ffa1caecca4db8f79fe665acdeaa5af815a24d
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:15 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:18 +0100

x86/speculation: Add basic IBPB (Indirect Branch Prediction Barrier) support

Expose indirect_branch_prediction_barrier() for use in subsequent patches.

[ tglx: Add IBPB status to spectre_v2 sysfs file ]

Co-developed-by: KarimAllah Ahmed 
Signed-off-by: KarimAllah Ahmed 
Signed-off-by: David Woodhouse 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-8-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/cpufeatures.h   |  2 ++
 arch/x86/include/asm/nospec-branch.h | 13 +
 arch/x86/kernel/cpu/bugs.c   | 10 +-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index ae3212f..07934b2 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -210,6 +210,8 @@
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
 #define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
 
+#define X86_FEATURE_IBPB   ( 7*32+21) /* Indirect Branch 
Prediction Barrier enabled*/
+
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
 #define X86_FEATURE_VNMI   ( 8*32+ 1) /* Intel Virtual NMI */
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 4ad4108..34e384c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -218,5 +218,18 @@ static inline void vmexit_fill_RSB(void)
 #endif
 }
 
+static inline void indirect_branch_prediction_barrier(void)
+{
+   asm volatile(ALTERNATIVE("",
+"movl %[msr], %%ecx\n\t"
+"movl %[val], %%eax\n\t"
+"movl $0, %%edx\n\t"
+"wrmsr",
+X86_FEATURE_IBPB)
+: : [msr] "i" (MSR_IA32_PRED_CMD),
+[val] "i" (PRED_CMD_IBPB)
+: "eax", "ecx", "edx", "memory");
+}
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 4a39d7b..bac7a35 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -263,6 +263,13 @@ retpoline_auto:
setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
pr_info("Filling RSB on context switch\n");
}
+
+   /* Initialize Indirect Branch Prediction Barrier if supported */
+   if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) ||
+   boot_cpu_has(X86_FEATURE_AMD_PRED_CMD)) {
+   setup_force_cpu_cap(X86_FEATURE_IBPB);
+   pr_info("Enabling Indirect Branch Prediction Barrier\n");
+   }
 }
 
 #undef pr_fmt
@@ -292,7 +299,8 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
return sprintf(buf, "Not affected\n");
 
-   return sprintf(buf, "%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+   return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+  boot_cpu_has(X86_FEATURE_IBPB) ? ", IPBP" : "",
   spectre_v2_bad_module ? " - vulnerable module loaded" : 
"");
 }
 #endif

[tip:x86/pti] x86/msr: Add definitions for new speculation control MSRs

[tip-bot for David Woodhouse]

Commit-ID:  1e340c60d0dd3ae07b5bedc16a0469c14b9f3410
Gitweb: https://git.kernel.org/tip/1e340c60d0dd3ae07b5bedc16a0469c14b9f3410
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:12 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:17 +0100

x86/msr: Add definitions for new speculation control MSRs

Add MSR and bit definitions for SPEC_CTRL, PRED_CMD and ARCH_CAPABILITIES.

See Intel's 336996-Speculative-Execution-Side-Channel-Mitigations.pdf

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-5-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/msr-index.h | 12 
 1 file changed, 12 insertions(+)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index fa11fb1..eb83ff1 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -39,6 +39,13 @@
 
 /* Intel MSRs. Some also available on other CPUs */
 
+#define MSR_IA32_SPEC_CTRL 0x0048 /* Speculation Control */
+#define SPEC_CTRL_IBRS (1 << 0)   /* Indirect Branch 
Restricted Speculation */
+#define SPEC_CTRL_STIBP(1 << 1)   /* Single Thread 
Indirect Branch Predictors */
+
+#define MSR_IA32_PRED_CMD  0x0049 /* Prediction Command */
+#define PRED_CMD_IBPB  (1 << 0)   /* Indirect Branch 
Prediction Barrier */
+
 #define MSR_PPIN_CTL   0x004e
 #define MSR_PPIN   0x004f
 
@@ -57,6 +64,11 @@
 #define SNB_C3_AUTO_UNDEMOTE   (1UL << 28)
 
 #define MSR_MTRRcap0x00fe
+
+#define MSR_IA32_ARCH_CAPABILITIES 0x010a
+#define ARCH_CAP_RDCL_NO   (1 << 0)   /* Not susceptible to 
Meltdown */
+#define ARCH_CAP_IBRS_ALL  (1 << 1)   /* Enhanced IBRS support */
+
 #define MSR_IA32_BBL_CR_CTL0x0119
 #define MSR_IA32_BBL_CR_CTL3   0x011e
 

[tip:x86/pti] x86/msr: Add definitions for new speculation control MSRs

[tip-bot for David Woodhouse]

Commit-ID:  1e340c60d0dd3ae07b5bedc16a0469c14b9f3410
Gitweb: https://git.kernel.org/tip/1e340c60d0dd3ae07b5bedc16a0469c14b9f3410
Author: David Woodhouse 
AuthorDate: Thu, 25 Jan 2018 16:14:12 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 26 Jan 2018 15:53:17 +0100

x86/msr: Add definitions for new speculation control MSRs

Add MSR and bit definitions for SPEC_CTRL, PRED_CMD and ARCH_CAPABILITIES.

See Intel's 336996-Speculative-Execution-Side-Channel-Mitigations.pdf

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Greg Kroah-Hartman 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: a...@linux.intel.com
Cc: ashok@intel.com
Cc: dave.han...@intel.com
Cc: karah...@amazon.de
Cc: ar...@linux.intel.com
Cc: torva...@linux-foundation.org
Cc: pet...@infradead.org
Cc: b...@alien8.de
Cc: pbonz...@redhat.com
Cc: tim.c.c...@linux.intel.com
Cc: gre...@linux-foundation.org
Link: 
https://lkml.kernel.org/r/1516896855-7642-5-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/msr-index.h | 12 
 1 file changed, 12 insertions(+)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index fa11fb1..eb83ff1 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -39,6 +39,13 @@
 
 /* Intel MSRs. Some also available on other CPUs */
 
+#define MSR_IA32_SPEC_CTRL 0x0048 /* Speculation Control */
+#define SPEC_CTRL_IBRS (1 << 0)   /* Indirect Branch 
Restricted Speculation */
+#define SPEC_CTRL_STIBP(1 << 1)   /* Single Thread 
Indirect Branch Predictors */
+
+#define MSR_IA32_PRED_CMD  0x0049 /* Prediction Command */
+#define PRED_CMD_IBPB  (1 << 0)   /* Indirect Branch 
Prediction Barrier */
+
 #define MSR_PPIN_CTL   0x004e
 #define MSR_PPIN   0x004f
 
@@ -57,6 +64,11 @@
 #define SNB_C3_AUTO_UNDEMOTE   (1UL << 28)
 
 #define MSR_MTRRcap0x00fe
+
+#define MSR_IA32_ARCH_CAPABILITIES 0x010a
+#define ARCH_CAP_RDCL_NO   (1 << 0)   /* Not susceptible to 
Meltdown */
+#define ARCH_CAP_IBRS_ALL  (1 << 1)   /* Enhanced IBRS support */
+
 #define MSR_IA32_BBL_CR_CTL0x0119
 #define MSR_IA32_BBL_CR_CTL3   0x011e
 

[tip:x86/pti] x86/retpoline: Fill RSB on context switch for affected CPUs

[tip-bot for David Woodhouse]

Commit-ID:  c995efd5a740d9cbafbf58bde4973e8b50b4d761
Gitweb: https://git.kernel.org/tip/c995efd5a740d9cbafbf58bde4973e8b50b4d761
Author: David Woodhouse 
AuthorDate: Fri, 12 Jan 2018 17:49:25 +
Committer:  Thomas Gleixner 
CommitDate: Mon, 15 Jan 2018 00:32:44 +0100

x86/retpoline: Fill RSB on context switch for affected CPUs

On context switch from a shallow call stack to a deeper one, as the CPU
does 'ret' up the deeper side it may encounter RSB entries (predictions for
where the 'ret' goes to) which were populated in userspace.

This is problematic if neither SMEP nor KPTI (the latter of which marks
userspace pages as NX for the kernel) are active, as malicious code in
userspace may then be executed speculatively.

Overwrite the CPU's return prediction stack with calls which are predicted
to return to an infinite loop, to "capture" speculation if this
happens. This is required both for retpoline, and also in conjunction with
IBRS for !SMEP && !KPTI.

On Skylake+ the problem is slightly different, and an *underflow* of the
RSB may cause errant branch predictions to occur. So there it's not so much
overwrite, as *filling* the RSB to attempt to prevent it getting
empty. This is only a partial solution for Skylake+ since there are many
other conditions which may result in the RSB becoming empty. The full
solution on Skylake+ is to use IBRS, which will prevent the problem even
when the RSB becomes empty. With IBRS, the RSB-stuffing will not be
required on context switch.

[ tglx: Added missing vendor check and slighty massaged comments and
changelog ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515779365-9032-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/entry/entry_32.S  | 11 +++
 arch/x86/entry/entry_64.S  | 11 +++
 arch/x86/include/asm/cpufeatures.h |  1 +
 arch/x86/kernel/cpu/bugs.c | 36 
 4 files changed, 59 insertions(+)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index a1f28a5..60c4c34 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -244,6 +244,17 @@ ENTRY(__switch_to_asm)
movl%ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popl%esi
popl%edi
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 59874bc..d54a0ed 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -487,6 +487,17 @@ ENTRY(__switch_to_asm)
movq%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popq%r15
popq%r14
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index f275447..aa09559 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 #define X86_FEATURE_AVX512_4FMAPS  ( 7*32+17) /* AVX-512 Multiply 
Accumulation Single precision */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
+#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/kernel/cpu/bugs.c 

[tip:x86/pti] x86/retpoline: Fill RSB on context switch for affected CPUs

[tip-bot for David Woodhouse]

Commit-ID:  c995efd5a740d9cbafbf58bde4973e8b50b4d761
Gitweb: https://git.kernel.org/tip/c995efd5a740d9cbafbf58bde4973e8b50b4d761
Author: David Woodhouse 
AuthorDate: Fri, 12 Jan 2018 17:49:25 +
Committer:  Thomas Gleixner 
CommitDate: Mon, 15 Jan 2018 00:32:44 +0100

x86/retpoline: Fill RSB on context switch for affected CPUs

On context switch from a shallow call stack to a deeper one, as the CPU
does 'ret' up the deeper side it may encounter RSB entries (predictions for
where the 'ret' goes to) which were populated in userspace.

This is problematic if neither SMEP nor KPTI (the latter of which marks
userspace pages as NX for the kernel) are active, as malicious code in
userspace may then be executed speculatively.

Overwrite the CPU's return prediction stack with calls which are predicted
to return to an infinite loop, to "capture" speculation if this
happens. This is required both for retpoline, and also in conjunction with
IBRS for !SMEP && !KPTI.

On Skylake+ the problem is slightly different, and an *underflow* of the
RSB may cause errant branch predictions to occur. So there it's not so much
overwrite, as *filling* the RSB to attempt to prevent it getting
empty. This is only a partial solution for Skylake+ since there are many
other conditions which may result in the RSB becoming empty. The full
solution on Skylake+ is to use IBRS, which will prevent the problem even
when the RSB becomes empty. With IBRS, the RSB-stuffing will not be
required on context switch.

[ tglx: Added missing vendor check and slighty massaged comments and
changelog ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515779365-9032-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/entry/entry_32.S  | 11 +++
 arch/x86/entry/entry_64.S  | 11 +++
 arch/x86/include/asm/cpufeatures.h |  1 +
 arch/x86/kernel/cpu/bugs.c | 36 
 4 files changed, 59 insertions(+)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index a1f28a5..60c4c34 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -244,6 +244,17 @@ ENTRY(__switch_to_asm)
movl%ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popl%esi
popl%edi
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 59874bc..d54a0ed 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -487,6 +487,17 @@ ENTRY(__switch_to_asm)
movq%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popq%r15
popq%r14
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index f275447..aa09559 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 #define X86_FEATURE_AVX512_4FMAPS  ( 7*32+17) /* AVX-512 Multiply 
Accumulation Single precision */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
+#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index e4dc261..390b3dc 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 
 static void __init spectre_v2_select_mitigation(void);
 
@@ -155,6 +156,23 @@ disable:
return SPECTRE_V2_CMD_NONE;
 }
 
+/* Check for Skylake-like CPUs (for RSB handling) */
+static 

[tip:x86/pti] x86/retpoline: Fill RSB on context switch for affected CPUs

[tip-bot for David Woodhouse]

Commit-ID:  a0ab15c0fb68e202bebd9b17fa49fd7ec48975b3
Gitweb: https://git.kernel.org/tip/a0ab15c0fb68e202bebd9b17fa49fd7ec48975b3
Author: David Woodhouse 
AuthorDate: Fri, 12 Jan 2018 17:49:25 +
Committer:  Thomas Gleixner 
CommitDate: Sun, 14 Jan 2018 16:41:39 +0100

x86/retpoline: Fill RSB on context switch for affected CPUs

On context switch from a shallow call stack to a deeper one, as the CPU
does 'ret' up the deeper side it may encounter RSB entries (predictions for
where the 'ret' goes to) which were populated in userspace.

This is problematic if neither SMEP nor KPTI (the latter of which marks
userspace pages as NX for the kernel) are active, as malicious code in
userspace may then be executed speculatively.

Overwrite the CPU's return prediction stack with calls which are predicted
to return to an infinite loop, to "capture" speculation if this
happens. This is required both for retpoline, and also in conjunction with
IBRS for !SMEP && !KPTI.

On Skylake+ the problem is slightly different, and an *underflow* of the
RSB may cause errant branch predictions to occur. So there it's not so much
overwrite, as *filling* the RSB to attempt to prevent it getting
empty. This is only a partial solution for Skylake+ since there are many
other conditions which may result in the RSB becoming empty. The full
solution on Skylake+ is to use IBRS, which will prevent the problem even
when the RSB becomes empty. With IBRS, the RSB-stuffing will not be
required on context switch.

[ tglx: Added missing vendor check and slighty massaged comments and
changelog ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515779365-9032-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/entry/entry_32.S  | 11 +++
 arch/x86/entry/entry_64.S  | 11 +++
 arch/x86/include/asm/cpufeatures.h |  1 +
 arch/x86/kernel/cpu/bugs.c | 36 
 4 files changed, 59 insertions(+)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index a1f28a5..60c4c34 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -244,6 +244,17 @@ ENTRY(__switch_to_asm)
movl%ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popl%esi
popl%edi
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 59874bc..d54a0ed 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -487,6 +487,17 @@ ENTRY(__switch_to_asm)
movq%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popq%r15
popq%r14
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index f275447..aa09559 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 #define X86_FEATURE_AVX512_4FMAPS  ( 7*32+17) /* AVX-512 Multiply 
Accumulation Single precision */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
+#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/kernel/cpu/bugs.c 

[tip:x86/pti] x86/retpoline: Fill RSB on context switch for affected CPUs

[tip-bot for David Woodhouse]

Commit-ID:  a0ab15c0fb68e202bebd9b17fa49fd7ec48975b3
Gitweb: https://git.kernel.org/tip/a0ab15c0fb68e202bebd9b17fa49fd7ec48975b3
Author: David Woodhouse 
AuthorDate: Fri, 12 Jan 2018 17:49:25 +
Committer:  Thomas Gleixner 
CommitDate: Sun, 14 Jan 2018 16:41:39 +0100

x86/retpoline: Fill RSB on context switch for affected CPUs

On context switch from a shallow call stack to a deeper one, as the CPU
does 'ret' up the deeper side it may encounter RSB entries (predictions for
where the 'ret' goes to) which were populated in userspace.

This is problematic if neither SMEP nor KPTI (the latter of which marks
userspace pages as NX for the kernel) are active, as malicious code in
userspace may then be executed speculatively.

Overwrite the CPU's return prediction stack with calls which are predicted
to return to an infinite loop, to "capture" speculation if this
happens. This is required both for retpoline, and also in conjunction with
IBRS for !SMEP && !KPTI.

On Skylake+ the problem is slightly different, and an *underflow* of the
RSB may cause errant branch predictions to occur. So there it's not so much
overwrite, as *filling* the RSB to attempt to prevent it getting
empty. This is only a partial solution for Skylake+ since there are many
other conditions which may result in the RSB becoming empty. The full
solution on Skylake+ is to use IBRS, which will prevent the problem even
when the RSB becomes empty. With IBRS, the RSB-stuffing will not be
required on context switch.

[ tglx: Added missing vendor check and slighty massaged comments and
changelog ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515779365-9032-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/entry/entry_32.S  | 11 +++
 arch/x86/entry/entry_64.S  | 11 +++
 arch/x86/include/asm/cpufeatures.h |  1 +
 arch/x86/kernel/cpu/bugs.c | 36 
 4 files changed, 59 insertions(+)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index a1f28a5..60c4c34 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -244,6 +244,17 @@ ENTRY(__switch_to_asm)
movl%ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popl%esi
popl%edi
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 59874bc..d54a0ed 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -487,6 +487,17 @@ ENTRY(__switch_to_asm)
movq%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
 #endif
 
+#ifdef CONFIG_RETPOLINE
+   /*
+* When switching from a shallower to a deeper call stack
+* the RSB may either underflow or use entries populated
+* with userspace addresses. On CPUs where those concerns
+* exist, overwrite the RSB with entries which capture
+* speculative execution to prevent attack.
+*/
+   FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
+#endif
+
/* restore callee-saved registers */
popq%r15
popq%r14
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index f275447..aa09559 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -211,6 +211,7 @@
 #define X86_FEATURE_AVX512_4FMAPS  ( 7*32+17) /* AVX-512 Multiply 
Accumulation Single precision */
 
 #define X86_FEATURE_MBA( 7*32+18) /* Memory Bandwidth 
Allocation */
+#define X86_FEATURE_RSB_CTXSW  ( 7*32+19) /* Fill RSB on context 
switches */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index e4dc261..390b3dc 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 
 static void __init spectre_v2_select_mitigation(void);
 
@@ -155,6 +156,23 @@ disable:
return SPECTRE_V2_CMD_NONE;
 }
 
+/* Check for Skylake-like CPUs (for RSB handling) */
+static 

[tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[tip-bot for David Woodhouse]

Commit-ID:  117cc7a908c83697b0b737d15ae1eb5943afe35b
Gitweb: https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
Author: David Woodhouse 
AuthorDate: Fri, 12 Jan 2018 11:11:27 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 12:33:37 +0100

x86/retpoline: Fill return stack buffer on vmexit

In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

[ak: numbers again for the RSB stuffing labels]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Tested-by: Peter Zijlstra (Intel) 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515755487-8524-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/nospec-branch.h | 78 +++-
 arch/x86/kvm/svm.c   |  4 ++
 arch/x86/kvm/vmx.c   |  4 ++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index ea034fa..402a11c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,48 @@
 #include 
 #include 
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include ") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS32  /* To forcibly overwrite all 
entries */
+#define RSB_FILL_LOOPS 16  /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)  \
+   mov $(nr/2), reg;   \
+771:   \
+   call772f;   \
+773:   /* speculation trap */  \
+   pause;  \
+   jmp 773b;   \
+772:   \
+   call774f;   \
+775:   /* speculation trap */  \
+   pause;  \
+   jmp 775b;   \
+774:   \
+   dec reg;\
+   jnz 771b;   \
+   add $(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +118,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+   ANNOTATE_NOSPEC_ALTERNATIVE
+   ALTERNATIVE "jmp .Lskip_rsb_\@",\
+   __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))\
+   \ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE\
@@ -119,7 +175,7 @@
X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation {
SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs 

[tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[tip-bot for David Woodhouse]

Commit-ID:  117cc7a908c83697b0b737d15ae1eb5943afe35b
Gitweb: https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
Author: David Woodhouse 
AuthorDate: Fri, 12 Jan 2018 11:11:27 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 12:33:37 +0100

x86/retpoline: Fill return stack buffer on vmexit

In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

[ak: numbers again for the RSB stuffing labels]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Tested-by: Peter Zijlstra (Intel) 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515755487-8524-1-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/nospec-branch.h | 78 +++-
 arch/x86/kvm/svm.c   |  4 ++
 arch/x86/kvm/vmx.c   |  4 ++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index ea034fa..402a11c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,48 @@
 #include 
 #include 
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include ") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS32  /* To forcibly overwrite all 
entries */
+#define RSB_FILL_LOOPS 16  /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)  \
+   mov $(nr/2), reg;   \
+771:   \
+   call772f;   \
+773:   /* speculation trap */  \
+   pause;  \
+   jmp 773b;   \
+772:   \
+   call774f;   \
+775:   /* speculation trap */  \
+   pause;  \
+   jmp 775b;   \
+774:   \
+   dec reg;\
+   jnz 771b;   \
+   add $(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +118,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+   ANNOTATE_NOSPEC_ALTERNATIVE
+   ALTERNATIVE "jmp .Lskip_rsb_\@",\
+   __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))\
+   \ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE\
@@ -119,7 +175,7 @@
X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation {
SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+   unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+   asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+ ALTERNATIVE("jmp 910f",
+ __stringify(__FILL_RETURN_BUFFER(%0, 
RSB_CLEAR_LOOPS, 

[tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[tip-bot for David Woodhouse]

Commit-ID:  85ec967c1dc04bde16d783ea04428bef3c00a171
Gitweb: https://git.kernel.org/tip/85ec967c1dc04bde16d783ea04428bef3c00a171
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:34 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:32 +0100

x86/retpoline: Fill return stack buffer on vmexit

In accordance with the Intel and AMD documentation, all entries in the RSB
must be overwrite on exiting a guest, to prevent malicious branch target
predictions from affecting the host kernel.

This is needed both for retpoline and for IBRS.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Tested-by: Peter Zijlstra (Intel) 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-13-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/nospec-branch.h | 73 +++-
 arch/x86/kvm/svm.c   |  4 ++
 arch/x86/kvm/vmx.c   |  4 ++
 3 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index ea034fa..475ab0c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,43 @@
 #include 
 #include 
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include ") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS32  /* To forcibly overwrite all 
entries */
+#define RSB_FILL_LOOPS 16  /* To avoid underflow */
+
+#define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)\
+   mov $(nr/2), reg;   \
+.Ldo_call1_ ## uniq:   \
+   call.Ldo_call2_ ## uniq;\
+.Ltrap1_ ## uniq:  \
+   pause;  \
+   jmp .Ltrap1_ ## uniq;   \
+.Ldo_call2_ ## uniq:   \
+   call.Ldo_loop_ ## uniq; \
+.Ltrap2_ ## uniq:  \
+   pause;  \
+   jmp .Ltrap2_ ## uniq;   \
+.Ldo_loop_ ## uniq:\
+   dec reg;\
+   jnz .Ldo_call1_ ## uniq;\
+   add $(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +113,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+   ANNOTATE_NOSPEC_ALTERNATIVE
+   ALTERNATIVE "jmp .Lskip_rsb_\@",\
+   __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP,\@)) \
+   \ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE\
@@ -119,7 +170,7 @@
X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +185,25 @@ enum spectre_v2_mitigation {
SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+   unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+   asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+ ALTERNATIVE("jmp 

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  ea08816d5b185ab3d09e95e393f265af54560350
Gitweb: https://git.kernel.org/tip/ea08816d5b185ab3d09e95e393f265af54560350
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:31 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:31 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Reviewed-by: Juergen Gross 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-10-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[tip-bot for David Woodhouse]

Commit-ID:  85ec967c1dc04bde16d783ea04428bef3c00a171
Gitweb: https://git.kernel.org/tip/85ec967c1dc04bde16d783ea04428bef3c00a171
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:34 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:32 +0100

x86/retpoline: Fill return stack buffer on vmexit

In accordance with the Intel and AMD documentation, all entries in the RSB
must be overwrite on exiting a guest, to prevent malicious branch target
predictions from affecting the host kernel.

This is needed both for retpoline and for IBRS.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Tested-by: Peter Zijlstra (Intel) 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-13-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/nospec-branch.h | 73 +++-
 arch/x86/kvm/svm.c   |  4 ++
 arch/x86/kvm/vmx.c   |  4 ++
 3 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index ea034fa..475ab0c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,43 @@
 #include 
 #include 
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include ") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS32  /* To forcibly overwrite all 
entries */
+#define RSB_FILL_LOOPS 16  /* To avoid underflow */
+
+#define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)\
+   mov $(nr/2), reg;   \
+.Ldo_call1_ ## uniq:   \
+   call.Ldo_call2_ ## uniq;\
+.Ltrap1_ ## uniq:  \
+   pause;  \
+   jmp .Ltrap1_ ## uniq;   \
+.Ldo_call2_ ## uniq:   \
+   call.Ldo_loop_ ## uniq; \
+.Ltrap2_ ## uniq:  \
+   pause;  \
+   jmp .Ltrap2_ ## uniq;   \
+.Ldo_loop_ ## uniq:\
+   dec reg;\
+   jnz .Ldo_call1_ ## uniq;\
+   add $(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +113,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+   ANNOTATE_NOSPEC_ALTERNATIVE
+   ALTERNATIVE "jmp .Lskip_rsb_\@",\
+   __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP,\@)) \
+   \ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE\
@@ -119,7 +170,7 @@
X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +185,25 @@ enum spectre_v2_mitigation {
SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+   unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+   asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+ ALTERNATIVE("jmp 910f",
+ __stringify(__FILL_RETURN_BUFFER(%0, 
RSB_CLEAR_LOOPS, %1, __LINE__)),
+ X86_FEATURE_RETPOLINE)
+ "910:"
+ : "=" (loops), ASM_CALL_CONSTRAINT
+ : "r" (loops) : "memory" );
+#endif
+}
 #endif /* __ASSEMBLY__ */
 #endif /* 

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  ea08816d5b185ab3d09e95e393f265af54560350
Gitweb: https://git.kernel.org/tip/ea08816d5b185ab3d09e95e393f265af54560350
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:31 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:31 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Reviewed-by: Juergen Gross 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-10-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  5096732f6f695001fa2d6f1335a2680b37912c69
Gitweb: https://git.kernel.org/tip/5096732f6f695001fa2d6f1335a2680b37912c69
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:32 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:31 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-11-git-send-email-d...@amazon.co.uk

---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  5096732f6f695001fa2d6f1335a2680b37912c69
Gitweb: https://git.kernel.org/tip/5096732f6f695001fa2d6f1335a2680b37912c69
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:32 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:31 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-11-git-send-email-d...@amazon.co.uk

---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  e70e5892b28c18f517f29ab6e83bd57705104b31
Gitweb: https://git.kernel.org/tip/e70e5892b28c18f517f29ab6e83bd57705104b31
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:30 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:30 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-9-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/mshyperv.h | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
return U64_MAX;
 
__asm__ __volatile__("mov %4, %%r8\n"
-"call *%5"
+CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input_address)
-:  "r" (output_address), "m" (hv_hypercall_pg)
+:  "r" (output_address),
+   THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
if (!hv_hypercall_pg)
return U64_MAX;
 
-   __asm__ __volatile__("call *%7"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=A" (hv_status),
   "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 : "A" (control),
   "b" (input_address_hi),
   "D"(output_address_hi), "S"(output_address_lo),
-  "m" (hv_hypercall_pg)
+  THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory");
 #endif /* !x86_64 */
return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
 
 #ifdef CONFIG_X86_64
{
-   __asm__ __volatile__("call *%4"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input1)
-: "m" (hv_hypercall_pg)
+: THUNK_TARGET(hv_hypercall_pg)
 : "cc", "r8", "r9", "r10", "r11");
}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
 
-   __asm__ __volatile__ ("call *%5"
+   __asm__ __volatile__ (CALL_NOSPEC
  : "=A"(hv_status),
"+c"(input1_lo),
ASM_CALL_CONSTRAINT
  : "A" (control),
"b" (input1_hi),
-   "m" (hv_hypercall_pg)
+   THUNK_TARGET(hv_hypercall_pg)
  : "cc", "edi", "esi");
}
 #endif

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  e70e5892b28c18f517f29ab6e83bd57705104b31
Gitweb: https://git.kernel.org/tip/e70e5892b28c18f517f29ab6e83bd57705104b31
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:30 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:30 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-9-git-send-email-d...@amazon.co.uk

---
 arch/x86/include/asm/mshyperv.h | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
return U64_MAX;
 
__asm__ __volatile__("mov %4, %%r8\n"
-"call *%5"
+CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input_address)
-:  "r" (output_address), "m" (hv_hypercall_pg)
+:  "r" (output_address),
+   THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
if (!hv_hypercall_pg)
return U64_MAX;
 
-   __asm__ __volatile__("call *%7"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=A" (hv_status),
   "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 : "A" (control),
   "b" (input_address_hi),
   "D"(output_address_hi), "S"(output_address_lo),
-  "m" (hv_hypercall_pg)
+  THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory");
 #endif /* !x86_64 */
return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
 
 #ifdef CONFIG_X86_64
{
-   __asm__ __volatile__("call *%4"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input1)
-: "m" (hv_hypercall_pg)
+: THUNK_TARGET(hv_hypercall_pg)
 : "cc", "r8", "r9", "r10", "r11");
}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
 
-   __asm__ __volatile__ ("call *%5"
+   __asm__ __volatile__ (CALL_NOSPEC
  : "=A"(hv_status),
"+c"(input1_lo),
ASM_CALL_CONSTRAINT
  : "A" (control),
"b" (input1_hi),
-   "m" (hv_hypercall_pg)
+   THUNK_TARGET(hv_hypercall_pg)
  : "cc", "edi", "esi");
}
 #endif

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9351803bd803cdbeb9b5a7850b7b6f464806e3db
Gitweb: https://git.kernel.org/tip/9351803bd803cdbeb9b5a7850b7b6f464806e3db
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:29 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:30 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-8-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/ftrace_32.S | 6 --
 arch/x86/kernel/ftrace_64.S | 8 
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef CC_USING_FENTRY
 # define function_hook __fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
movl0x4(%ebp), %edx
subl$MCOUNT_INSN_SIZE, %eax
 
-   call*ftrace_trace_function
+   movlftrace_trace_function, %ecx
+   CALL_NOSPEC %ecx
 
popl%edx
popl%ecx
@@ -241,5 +243,5 @@ return_to_handler:
movl%eax, %ecx
popl%edx
popl%eax
-   jmp *%ecx
+   JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include 
 #include 
 #include 
-
+#include 
 
.code64
.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 * ip and parent ip are used and the list function is called when
 * function tracing is enabled.
 */
-   call   *ftrace_trace_function
-
+   movq ftrace_trace_function, %r8
+   CALL_NOSPEC %r8
restore_mcount_regs
 
jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 #endif

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9351803bd803cdbeb9b5a7850b7b6f464806e3db
Gitweb: https://git.kernel.org/tip/9351803bd803cdbeb9b5a7850b7b6f464806e3db
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:29 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:30 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-8-git-send-email-d...@amazon.co.uk

---
 arch/x86/kernel/ftrace_32.S | 6 --
 arch/x86/kernel/ftrace_64.S | 8 
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef CC_USING_FENTRY
 # define function_hook __fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
movl0x4(%ebp), %edx
subl$MCOUNT_INSN_SIZE, %eax
 
-   call*ftrace_trace_function
+   movlftrace_trace_function, %ecx
+   CALL_NOSPEC %ecx
 
popl%edx
popl%ecx
@@ -241,5 +243,5 @@ return_to_handler:
movl%eax, %ecx
popl%edx
popl%eax
-   jmp *%ecx
+   JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include 
 #include 
 #include 
-
+#include 
 
.code64
.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 * ip and parent ip are used and the list function is called when
 * function tracing is enabled.
 */
-   call   *ftrace_trace_function
-
+   movq ftrace_trace_function, %r8
+   CALL_NOSPEC %r8
restore_mcount_regs
 
jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 #endif

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9697fa39efd3fc3692f2949d4045f393ec58450b
Gitweb: https://git.kernel.org/tip/9697fa39efd3fc3692f2949d4045f393ec58450b
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:27 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-6-git-send-email-d...@amazon.co.uk

---
 arch/x86/crypto/aesni-intel_asm.S| 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S| 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S 
b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
pxor INC, STATE4
movdqu IV, 0x30(OUTP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x00(OUTP), INC
pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
_aesni_gf128mul_x_ble()
movups IV, (IVP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x40(OUTP), INC
pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
vpxor 14 * 16(%rax), %xmm15, %xmm14;
vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
vpxor 14 * 32(%rax), %ymm15, %ymm14;
vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S 
b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include 
 #include 
+#include 
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
movzxw  (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
-   jmp *bufp
+   JMP_NOSPEC bufp
 

## 2a) PROCESS FULL BLOCKS:

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9697fa39efd3fc3692f2949d4045f393ec58450b
Gitweb: https://git.kernel.org/tip/9697fa39efd3fc3692f2949d4045f393ec58450b
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:27 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-6-git-send-email-d...@amazon.co.uk

---
 arch/x86/crypto/aesni-intel_asm.S| 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S| 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S 
b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
pxor INC, STATE4
movdqu IV, 0x30(OUTP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x00(OUTP), INC
pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
_aesni_gf128mul_x_ble()
movups IV, (IVP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x40(OUTP), INC
pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
vpxor 14 * 16(%rax), %xmm15, %xmm14;
vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
vpxor 14 * 32(%rax), %ymm15, %ymm14;
vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S 
b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include 
 #include 
+#include 
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
movzxw  (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
-   jmp *bufp
+   JMP_NOSPEC bufp
 

## 2a) PROCESS FULL BLOCKS:

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  2641f08bb7fc63a636a2b18173221d7040a3512e
Gitweb: https://git.kernel.org/tip/2641f08bb7fc63a636a2b18173221d7040a3512e
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:28 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Ingo Molnar 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-7-git-send-email-d...@amazon.co.uk

---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
/* kernel thread */
 1: movl%edi, %eax
-   call*%ebx
+   CALL_NOSPEC %ebx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
movl%ecx, %es
TRACE_IRQS_OFF
movl%esp, %eax  # pt_regs pointer
-   call*%edi
+   CALL_NOSPEC %edi
jmp ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 */
pushq   %rdi
movq$entry_SYSCALL_64_stage2, %rdi
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 * It might end up jumping to the slow path.  If it jumps, RAX
 * and all argument registers are clobbered.
 */
+#ifdef CONFIG_RETPOLINE
+   movqsys_call_table(, %rax, 8), %rax
+   call__x86_indirect_thunk_rax
+#else
call*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
movq%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
jmp entry_SYSCALL64_slow_path
 
 1:
-   jmp *%rax   /* Called from C */
+   JMP_NOSPEC %rax /* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
/* kernel thread */
movq%r12, %rdi
-   call*%rbx
+   CALL_NOSPEC %rbx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  2641f08bb7fc63a636a2b18173221d7040a3512e
Gitweb: https://git.kernel.org/tip/2641f08bb7fc63a636a2b18173221d7040a3512e
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:28 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Ingo Molnar 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-7-git-send-email-d...@amazon.co.uk

---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
/* kernel thread */
 1: movl%edi, %eax
-   call*%ebx
+   CALL_NOSPEC %ebx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
movl%ecx, %es
TRACE_IRQS_OFF
movl%esp, %eax  # pt_regs pointer
-   call*%edi
+   CALL_NOSPEC %edi
jmp ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 */
pushq   %rdi
movq$entry_SYSCALL_64_stage2, %rdi
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 * It might end up jumping to the slow path.  If it jumps, RAX
 * and all argument registers are clobbered.
 */
+#ifdef CONFIG_RETPOLINE
+   movqsys_call_table(, %rax, 8), %rax
+   call__x86_indirect_thunk_rax
+#else
call*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
movq%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
jmp entry_SYSCALL64_slow_path
 
 1:
-   jmp *%rax   /* Called from C */
+   JMP_NOSPEC %rax /* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
/* kernel thread */
movq%r12, %rdi
-   call*%rbx
+   CALL_NOSPEC %rbx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  76b043848fd22dbf7f8bf3a1452f8c70d557b860
Gitweb: https://git.kernel.org/tip/76b043848fd22dbf7f8bf3a1452f8c70d557b860
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:25 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:28 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-d...@amazon.co.uk

---
 arch/x86/Kconfig  |  13 
 arch/x86/Makefile |  10 +++
 arch/x86/include/asm/asm-prototypes.h |  25 +++
 arch/x86/include/asm/cpufeatures.h|   2 +
 arch/x86/include/asm/nospec-branch.h  | 128 ++
 arch/x86/kernel/cpu/common.c  |   4 ++
 arch/x86/lib/Makefile |   1 +
 arch/x86/lib/retpoline.S  |  48 +
 8 files changed, 231 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
def_bool y
depends on X86_GOLDFISH
 
+config RETPOLINE
+   bool "Avoid speculative indirect branches in kernel"
+   default y
+   help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
 config INTEL_RDT
bool "Intel Resource Director Technology support"
default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+ifneq ($(RETPOLINE_CFLAGS),)
+KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+else
+$(warning CONFIG_RETPOLINE=y, but not supported by the compiler. 
Toolchain update recommended.)
+endif
+endif
+
 archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## 
reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## 

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  76b043848fd22dbf7f8bf3a1452f8c70d557b860
Gitweb: https://git.kernel.org/tip/76b043848fd22dbf7f8bf3a1452f8c70d557b860
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:25 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:28 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-d...@amazon.co.uk

---
 arch/x86/Kconfig  |  13 
 arch/x86/Makefile |  10 +++
 arch/x86/include/asm/asm-prototypes.h |  25 +++
 arch/x86/include/asm/cpufeatures.h|   2 +
 arch/x86/include/asm/nospec-branch.h  | 128 ++
 arch/x86/kernel/cpu/common.c  |   4 ++
 arch/x86/lib/Makefile |   1 +
 arch/x86/lib/retpoline.S  |  48 +
 8 files changed, 231 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
def_bool y
depends on X86_GOLDFISH
 
+config RETPOLINE
+   bool "Avoid speculative indirect branches in kernel"
+   default y
+   help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
 config INTEL_RDT
bool "Intel Resource Director Technology support"
default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+ifneq ($(RETPOLINE_CFLAGS),)
+KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+else
+$(warning CONFIG_RETPOLINE=y, but not supported by the compiler. 
Toolchain update recommended.)
+endif
+endif
+
 archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## 
reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## 
reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git 

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  da285121560e769cc31797bba6422eea71d473e0
Gitweb: https://git.kernel.org/tip/da285121560e769cc31797bba6422eea71d473e0
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:26 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
integration becomes simple ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-5-git-send-email-d...@amazon.co.uk

---
 Documentation/admin-guide/kernel-parameters.txt |  28 +
 arch/x86/include/asm/nospec-branch.h|  10 ++
 arch/x86/kernel/cpu/bugs.c  | 158 +++-
 arch/x86/kernel/cpu/common.c|   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt 
b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
nosmt   [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
 
+   nospectre_v2[X86] Disable all mitigations for the Spectre variant 2
+   (indirect branch prediction) vulnerability. System may
+   allow data leaks with this option, which is equivalent
+   to spectre_v2=off.
+
noxsave [BUGS=X86] Disables x86 extended register state save
and restore using xsave. The kernel will fallback to
enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
sonypi.*=   [HW] Sony Programmable I/O Control Device driver
See Documentation/laptops/sonypi.txt
 
+   spectre_v2= [X86] Control mitigation of Spectre variant 2
+   (indirect branch speculation) vulnerability.
+
+   on   - unconditionally enable
+   off  - unconditionally disable
+   auto - kernel detects whether your CPU model is
+  vulnerable
+
+   Selecting 'on' will, and 'auto' may, choose a
+   mitigation method at run time according to the
+   CPU, the available microcode, the setting of the
+   CONFIG_RETPOLINE configuration option, and the
+   compiler with which the kernel was built.
+
+   Specific mitigations can also be selected manually:
+
+   retpoline - replace indirect branches
+   retpoline,generic - google's original retpoline
+   retpoline,amd - AMD-specific minimal thunk
+
+   Not specifying this option is equivalent to
+   spectre_v2=auto.
+
spia_io_base=   [HW,MTD]
spia_fio_base=
spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index e20e92e..ea034fa 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -124,5 +124,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+   SPECTRE_V2_NONE,
+   SPECTRE_V2_RETPOLINE_MINIMAL,
+   SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+   SPECTRE_V2_RETPOLINE_GENERIC,
+   SPECTRE_V2_RETPOLINE_AMD,
+   SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..e4dc261 100644
--- 

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  da285121560e769cc31797bba6422eea71d473e0
Gitweb: https://git.kernel.org/tip/da285121560e769cc31797bba6422eea71d473e0
Author: David Woodhouse 
AuthorDate: Thu, 11 Jan 2018 21:46:26 +
Committer:  Thomas Gleixner 
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
integration becomes simple ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Josh Poimboeuf 
Cc: thomas.lenda...@amd.com
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515707194-20531-5-git-send-email-d...@amazon.co.uk

---
 Documentation/admin-guide/kernel-parameters.txt |  28 +
 arch/x86/include/asm/nospec-branch.h|  10 ++
 arch/x86/kernel/cpu/bugs.c  | 158 +++-
 arch/x86/kernel/cpu/common.c|   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt 
b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
nosmt   [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
 
+   nospectre_v2[X86] Disable all mitigations for the Spectre variant 2
+   (indirect branch prediction) vulnerability. System may
+   allow data leaks with this option, which is equivalent
+   to spectre_v2=off.
+
noxsave [BUGS=X86] Disables x86 extended register state save
and restore using xsave. The kernel will fallback to
enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
sonypi.*=   [HW] Sony Programmable I/O Control Device driver
See Documentation/laptops/sonypi.txt
 
+   spectre_v2= [X86] Control mitigation of Spectre variant 2
+   (indirect branch speculation) vulnerability.
+
+   on   - unconditionally enable
+   off  - unconditionally disable
+   auto - kernel detects whether your CPU model is
+  vulnerable
+
+   Selecting 'on' will, and 'auto' may, choose a
+   mitigation method at run time according to the
+   CPU, the available microcode, the setting of the
+   CONFIG_RETPOLINE configuration option, and the
+   compiler with which the kernel was built.
+
+   Specific mitigations can also be selected manually:
+
+   retpoline - replace indirect branches
+   retpoline,generic - google's original retpoline
+   retpoline,amd - AMD-specific minimal thunk
+
+   Not specifying this option is equivalent to
+   spectre_v2=auto.
+
spia_io_base=   [HW,MTD]
spia_fio_base=
spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index e20e92e..ea034fa 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -124,5 +124,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+   SPECTRE_V2_NONE,
+   SPECTRE_V2_RETPOLINE_MINIMAL,
+   SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+   SPECTRE_V2_RETPOLINE_GENERIC,
+   SPECTRE_V2_RETPOLINE_AMD,
+   SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..e4dc261 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -11,6 +11,9 @@
 #include 
 #include 
 #include 
+
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -21,6 +24,8 @@
 #include 
 #include 
 
+static void __init spectre_v2_select_mitigation(void);
+
 void __init check_bugs(void)
 {
identify_boot_cpu();
@@ -30,6 +35,9 @@ 

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  96f71b3a482e918991d165eb7a6b42eb9a9ef735
Gitweb: https://git.kernel.org/tip/96f71b3a482e918991d165eb7a6b42eb9a9ef735
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:15 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:25 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-d...@amazon.co.uk


---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  96f71b3a482e918991d165eb7a6b42eb9a9ef735
Gitweb: https://git.kernel.org/tip/96f71b3a482e918991d165eb7a6b42eb9a9ef735
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:15 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:25 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-d...@amazon.co.uk


---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  b569cb1e72bda00e7e6245519fe7d0d0ab13898e
Gitweb: https://git.kernel.org/tip/b569cb1e72bda00e7e6245519fe7d0d0ab13898e
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:14 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:25 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Juergen Gross 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  b569cb1e72bda00e7e6245519fe7d0d0ab13898e
Gitweb: https://git.kernel.org/tip/b569cb1e72bda00e7e6245519fe7d0d0ab13898e
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:14 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:25 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Juergen Gross 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  f14fd95d2f3e611619756ea3c008aee3b4bd4978
Gitweb: https://git.kernel.org/tip/f14fd95d2f3e611619756ea3c008aee3b4bd4978
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:13 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:25 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-8-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/mshyperv.h | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
return U64_MAX;
 
__asm__ __volatile__("mov %4, %%r8\n"
-"call *%5"
+CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input_address)
-:  "r" (output_address), "m" (hv_hypercall_pg)
+:  "r" (output_address),
+   THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
if (!hv_hypercall_pg)
return U64_MAX;
 
-   __asm__ __volatile__("call *%7"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=A" (hv_status),
   "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 : "A" (control),
   "b" (input_address_hi),
   "D"(output_address_hi), "S"(output_address_lo),
-  "m" (hv_hypercall_pg)
+  THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory");
 #endif /* !x86_64 */
return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
 
 #ifdef CONFIG_X86_64
{
-   __asm__ __volatile__("call *%4"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input1)
-: "m" (hv_hypercall_pg)
+: THUNK_TARGET(hv_hypercall_pg)
 : "cc", "r8", "r9", "r10", "r11");
}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
 
-   __asm__ __volatile__ ("call *%5"
+   __asm__ __volatile__ (CALL_NOSPEC
  : "=A"(hv_status),
"+c"(input1_lo),
ASM_CALL_CONSTRAINT
  : "A" (control),
"b" (input1_hi),
-   "m" (hv_hypercall_pg)
+   THUNK_TARGET(hv_hypercall_pg)
  : "cc", "edi", "esi");
}
 #endif

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  f14fd95d2f3e611619756ea3c008aee3b4bd4978
Gitweb: https://git.kernel.org/tip/f14fd95d2f3e611619756ea3c008aee3b4bd4978
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:13 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:25 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-8-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/mshyperv.h | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
return U64_MAX;
 
__asm__ __volatile__("mov %4, %%r8\n"
-"call *%5"
+CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input_address)
-:  "r" (output_address), "m" (hv_hypercall_pg)
+:  "r" (output_address),
+   THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
if (!hv_hypercall_pg)
return U64_MAX;
 
-   __asm__ __volatile__("call *%7"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=A" (hv_status),
   "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 : "A" (control),
   "b" (input_address_hi),
   "D"(output_address_hi), "S"(output_address_lo),
-  "m" (hv_hypercall_pg)
+  THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory");
 #endif /* !x86_64 */
return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
 
 #ifdef CONFIG_X86_64
{
-   __asm__ __volatile__("call *%4"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input1)
-: "m" (hv_hypercall_pg)
+: THUNK_TARGET(hv_hypercall_pg)
 : "cc", "r8", "r9", "r10", "r11");
}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
 
-   __asm__ __volatile__ ("call *%5"
+   __asm__ __volatile__ (CALL_NOSPEC
  : "=A"(hv_status),
"+c"(input1_lo),
ASM_CALL_CONSTRAINT
  : "A" (control),
"b" (input1_hi),
-   "m" (hv_hypercall_pg)
+   THUNK_TARGET(hv_hypercall_pg)
  : "cc", "edi", "esi");
}
 #endif

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  2558106c7a47e16968a10fa66eea78a096fabfe6
Gitweb: https://git.kernel.org/tip/2558106c7a47e16968a10fa66eea78a096fabfe6
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:11 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:24 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Ingo Molnar 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-6-git-send-email-d...@amazon.co.uk


---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
/* kernel thread */
 1: movl%edi, %eax
-   call*%ebx
+   CALL_NOSPEC %ebx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
movl%ecx, %es
TRACE_IRQS_OFF
movl%esp, %eax  # pt_regs pointer
-   call*%edi
+   CALL_NOSPEC %edi
jmp ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 */
pushq   %rdi
movq$entry_SYSCALL_64_stage2, %rdi
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 * It might end up jumping to the slow path.  If it jumps, RAX
 * and all argument registers are clobbered.
 */
+#ifdef CONFIG_RETPOLINE
+   movqsys_call_table(, %rax, 8), %rax
+   call__x86_indirect_thunk_rax
+#else
call*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
movq%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
jmp entry_SYSCALL64_slow_path
 
 1:
-   jmp *%rax   /* Called from C */
+   JMP_NOSPEC %rax /* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
/* kernel thread */
movq%r12, %rdi
-   call*%rbx
+   CALL_NOSPEC %rbx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  2558106c7a47e16968a10fa66eea78a096fabfe6
Gitweb: https://git.kernel.org/tip/2558106c7a47e16968a10fa66eea78a096fabfe6
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:11 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:24 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Ingo Molnar 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-6-git-send-email-d...@amazon.co.uk


---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
/* kernel thread */
 1: movl%edi, %eax
-   call*%ebx
+   CALL_NOSPEC %ebx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
movl%ecx, %es
TRACE_IRQS_OFF
movl%esp, %eax  # pt_regs pointer
-   call*%edi
+   CALL_NOSPEC %edi
jmp ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 */
pushq   %rdi
movq$entry_SYSCALL_64_stage2, %rdi
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 * It might end up jumping to the slow path.  If it jumps, RAX
 * and all argument registers are clobbered.
 */
+#ifdef CONFIG_RETPOLINE
+   movqsys_call_table(, %rax, 8), %rax
+   call__x86_indirect_thunk_rax
+#else
call*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
movq%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
jmp entry_SYSCALL64_slow_path
 
 1:
-   jmp *%rax   /* Called from C */
+   JMP_NOSPEC %rax /* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
/* kernel thread */
movq%r12, %rdi
-   call*%rbx
+   CALL_NOSPEC %rbx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  42f7c812022441ffba2d5ccca3acf6380201f19e
Gitweb: https://git.kernel.org/tip/42f7c812022441ffba2d5ccca3acf6380201f19e
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:12 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:24 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-7-git-send-email-d...@amazon.co.uk


---
 arch/x86/kernel/ftrace_32.S | 6 --
 arch/x86/kernel/ftrace_64.S | 8 
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef CC_USING_FENTRY
 # define function_hook __fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
movl0x4(%ebp), %edx
subl$MCOUNT_INSN_SIZE, %eax
 
-   call*ftrace_trace_function
+   movlftrace_trace_function, %ecx
+   CALL_NOSPEC %ecx
 
popl%edx
popl%ecx
@@ -241,5 +243,5 @@ return_to_handler:
movl%eax, %ecx
popl%edx
popl%eax
-   jmp *%ecx
+   JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include 
 #include 
 #include 
-
+#include 
 
.code64
.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 * ip and parent ip are used and the list function is called when
 * function tracing is enabled.
 */
-   call   *ftrace_trace_function
-
+   movq ftrace_trace_function, %r8
+   CALL_NOSPEC %r8
restore_mcount_regs
 
jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 #endif

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  42f7c812022441ffba2d5ccca3acf6380201f19e
Gitweb: https://git.kernel.org/tip/42f7c812022441ffba2d5ccca3acf6380201f19e
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:12 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:24 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-7-git-send-email-d...@amazon.co.uk


---
 arch/x86/kernel/ftrace_32.S | 6 --
 arch/x86/kernel/ftrace_64.S | 8 
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef CC_USING_FENTRY
 # define function_hook __fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
movl0x4(%ebp), %edx
subl$MCOUNT_INSN_SIZE, %eax
 
-   call*ftrace_trace_function
+   movlftrace_trace_function, %ecx
+   CALL_NOSPEC %ecx
 
popl%edx
popl%ecx
@@ -241,5 +243,5 @@ return_to_handler:
movl%eax, %ecx
popl%edx
popl%eax
-   jmp *%ecx
+   JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include 
 #include 
 #include 
-
+#include 
 
.code64
.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 * ip and parent ip are used and the list function is called when
 * function tracing is enabled.
 */
-   call   *ftrace_trace_function
-
+   movq ftrace_trace_function, %r8
+   CALL_NOSPEC %r8
restore_mcount_regs
 
jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 #endif

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  b3a96862283e68914d1f74f160ab980dacf811ee
Gitweb: https://git.kernel.org/tip/b3a96862283e68914d1f74f160ab980dacf811ee
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:10 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:24 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-5-git-send-email-d...@amazon.co.uk


---
 arch/x86/crypto/aesni-intel_asm.S| 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S| 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S 
b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
pxor INC, STATE4
movdqu IV, 0x30(OUTP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x00(OUTP), INC
pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
_aesni_gf128mul_x_ble()
movups IV, (IVP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x40(OUTP), INC
pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
vpxor 14 * 16(%rax), %xmm15, %xmm14;
vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
vpxor 14 * 32(%rax), %ymm15, %ymm14;
vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S 
b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include 
 #include 
+#include 
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
movzxw  (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
-   jmp *bufp
+   JMP_NOSPEC bufp
 

## 2a) PROCESS FULL BLOCKS:

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  b3a96862283e68914d1f74f160ab980dacf811ee
Gitweb: https://git.kernel.org/tip/b3a96862283e68914d1f74f160ab980dacf811ee
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:10 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:24 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-5-git-send-email-d...@amazon.co.uk


---
 arch/x86/crypto/aesni-intel_asm.S| 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S| 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S 
b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
pxor INC, STATE4
movdqu IV, 0x30(OUTP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x00(OUTP), INC
pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
_aesni_gf128mul_x_ble()
movups IV, (IVP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x40(OUTP), INC
pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
vpxor 14 * 16(%rax), %xmm15, %xmm14;
vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
vpxor 14 * 32(%rax), %ymm15, %ymm14;
vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S 
b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include 
 #include 
+#include 
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
movzxw  (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
-   jmp *bufp
+   JMP_NOSPEC bufp
 

## 2a) PROCESS FULL BLOCKS:

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  86d057614112971f7d5bbac45f67869adca79852
Gitweb: https://git.kernel.org/tip/86d057614112971f7d5bbac45f67869adca79852
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:09 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:23 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
integration becomes simple ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Cc: Tom Lendacky 
Link: 
https://lkml.kernel.org/r/1515508997-6154-4-git-send-email-d...@amazon.co.uk


---
 Documentation/admin-guide/kernel-parameters.txt |  28 +
 arch/x86/include/asm/nospec-branch.h|  10 ++
 arch/x86/kernel/cpu/bugs.c  | 158 +++-
 arch/x86/kernel/cpu/common.c|   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt 
b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
nosmt   [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
 
+   nospectre_v2[X86] Disable all mitigations for the Spectre variant 2
+   (indirect branch prediction) vulnerability. System may
+   allow data leaks with this option, which is equivalent
+   to spectre_v2=off.
+
noxsave [BUGS=X86] Disables x86 extended register state save
and restore using xsave. The kernel will fallback to
enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
sonypi.*=   [HW] Sony Programmable I/O Control Device driver
See Documentation/laptops/sonypi.txt
 
+   spectre_v2= [X86] Control mitigation of Spectre variant 2
+   (indirect branch speculation) vulnerability.
+
+   on   - unconditionally enable
+   off  - unconditionally disable
+   auto - kernel detects whether your CPU model is
+  vulnerable
+
+   Selecting 'on' will, and 'auto' may, choose a
+   mitigation method at run time according to the
+   CPU, the available microcode, the setting of the
+   CONFIG_RETPOLINE configuration option, and the
+   compiler with which the kernel was built.
+
+   Specific mitigations can also be selected manually:
+
+   retpoline - replace indirect branches
+   retpoline,generic - google's original retpoline
+   retpoline,amd - AMD-specific minimal thunk
+
+   Not specifying this option is equivalent to
+   spectre_v2=auto.
+
spia_io_base=   [HW,MTD]
spia_fio_base=
spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 7f58713..7d70ea9 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -105,5 +105,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+   SPECTRE_V2_NONE,
+   SPECTRE_V2_RETPOLINE_MINIMAL,
+   SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+   SPECTRE_V2_RETPOLINE_GENERIC,
+   SPECTRE_V2_RETPOLINE_AMD,
+   SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..e4dc261 100644
--- a/arch/x86/kernel/cpu/bugs.c

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  86d057614112971f7d5bbac45f67869adca79852
Gitweb: https://git.kernel.org/tip/86d057614112971f7d5bbac45f67869adca79852
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:09 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:23 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
integration becomes simple ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Cc: Tom Lendacky 
Link: 
https://lkml.kernel.org/r/1515508997-6154-4-git-send-email-d...@amazon.co.uk


---
 Documentation/admin-guide/kernel-parameters.txt |  28 +
 arch/x86/include/asm/nospec-branch.h|  10 ++
 arch/x86/kernel/cpu/bugs.c  | 158 +++-
 arch/x86/kernel/cpu/common.c|   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt 
b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
nosmt   [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
 
+   nospectre_v2[X86] Disable all mitigations for the Spectre variant 2
+   (indirect branch prediction) vulnerability. System may
+   allow data leaks with this option, which is equivalent
+   to spectre_v2=off.
+
noxsave [BUGS=X86] Disables x86 extended register state save
and restore using xsave. The kernel will fallback to
enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
sonypi.*=   [HW] Sony Programmable I/O Control Device driver
See Documentation/laptops/sonypi.txt
 
+   spectre_v2= [X86] Control mitigation of Spectre variant 2
+   (indirect branch speculation) vulnerability.
+
+   on   - unconditionally enable
+   off  - unconditionally disable
+   auto - kernel detects whether your CPU model is
+  vulnerable
+
+   Selecting 'on' will, and 'auto' may, choose a
+   mitigation method at run time according to the
+   CPU, the available microcode, the setting of the
+   CONFIG_RETPOLINE configuration option, and the
+   compiler with which the kernel was built.
+
+   Specific mitigations can also be selected manually:
+
+   retpoline - replace indirect branches
+   retpoline,generic - google's original retpoline
+   retpoline,amd - AMD-specific minimal thunk
+
+   Not specifying this option is equivalent to
+   spectre_v2=auto.
+
spia_io_base=   [HW,MTD]
spia_fio_base=
spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 7f58713..7d70ea9 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -105,5 +105,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+   SPECTRE_V2_NONE,
+   SPECTRE_V2_RETPOLINE_MINIMAL,
+   SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+   SPECTRE_V2_RETPOLINE_GENERIC,
+   SPECTRE_V2_RETPOLINE_AMD,
+   SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..e4dc261 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -11,6 +11,9 @@
 #include 
 #include 
 #include 
+
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -21,6 +24,8 @@
 #include 
 #include 
 
+static void __init spectre_v2_select_mitigation(void);
+
 void __init check_bugs(void)
 {
identify_boot_cpu();
@@ -30,6 +35,9 @@ void __init check_bugs(void)
  

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  d46717c610dcfa2cba5c87500c928993371ef1ad
Gitweb: https://git.kernel.org/tip/d46717c610dcfa2cba5c87500c928993371ef1ad
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:07 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:22 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-d...@amazon.co.uk


---
 arch/x86/Kconfig  |  13 
 arch/x86/Makefile |  10 
 arch/x86/include/asm/asm-prototypes.h |  25 
 arch/x86/include/asm/cpufeatures.h|   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++
 arch/x86/kernel/cpu/common.c  |   4 ++
 arch/x86/lib/Makefile |   1 +
 arch/x86/lib/retpoline.S  |  48 +++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
def_bool y
depends on X86_GOLDFISH
 
+config RETPOLINE
+   bool "Avoid speculative indirect branches in kernel"
+   default y
+   help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
 config INTEL_RDT
bool "Intel Resource Director Technology support"
default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+ifneq ($(RETPOLINE_CFLAGS),)
+KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+else
+$(warning CONFIG_RETPOLINE=y, but not supported by the compiler. 
Toolchain update recommended.)
+endif
+endif
+
 archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## 
reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## 
reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  d46717c610dcfa2cba5c87500c928993371ef1ad
Gitweb: https://git.kernel.org/tip/d46717c610dcfa2cba5c87500c928993371ef1ad
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:07 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:36:22 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-d...@amazon.co.uk


---
 arch/x86/Kconfig  |  13 
 arch/x86/Makefile |  10 
 arch/x86/include/asm/asm-prototypes.h |  25 
 arch/x86/include/asm/cpufeatures.h|   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++
 arch/x86/kernel/cpu/common.c  |   4 ++
 arch/x86/lib/Makefile |   1 +
 arch/x86/lib/retpoline.S  |  48 +++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
def_bool y
depends on X86_GOLDFISH
 
+config RETPOLINE
+   bool "Avoid speculative indirect branches in kernel"
+   default y
+   help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
 config INTEL_RDT
bool "Intel Resource Director Technology support"
default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+ifneq ($(RETPOLINE_CFLAGS),)
+KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+else
+$(warning CONFIG_RETPOLINE=y, but not supported by the compiler. 
Toolchain update recommended.)
+endif
+endif
+
 archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## 
reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## 
reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- 

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  689e1c6117fcba2fb4b1f30d164a7a80389c0b29
Gitweb: https://git.kernel.org/tip/689e1c6117fcba2fb4b1f30d164a7a80389c0b29
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:12 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:11 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-7-git-send-email-d...@amazon.co.uk


---
 arch/x86/kernel/ftrace_32.S | 6 --
 arch/x86/kernel/ftrace_64.S | 8 
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef CC_USING_FENTRY
 # define function_hook __fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
movl0x4(%ebp), %edx
subl$MCOUNT_INSN_SIZE, %eax
 
-   call*ftrace_trace_function
+   movlftrace_trace_function, %ecx
+   CALL_NOSPEC %ecx
 
popl%edx
popl%ecx
@@ -241,5 +243,5 @@ return_to_handler:
movl%eax, %ecx
popl%edx
popl%eax
-   jmp *%ecx
+   JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include 
 #include 
 #include 
-
+#include 
 
.code64
.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 * ip and parent ip are used and the list function is called when
 * function tracing is enabled.
 */
-   call   *ftrace_trace_function
-
+   movq ftrace_trace_function, %r8
+   CALL_NOSPEC %r8
restore_mcount_regs
 
jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 #endif

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  689e1c6117fcba2fb4b1f30d164a7a80389c0b29
Gitweb: https://git.kernel.org/tip/689e1c6117fcba2fb4b1f30d164a7a80389c0b29
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:12 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:11 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-7-git-send-email-d...@amazon.co.uk


---
 arch/x86/kernel/ftrace_32.S | 6 --
 arch/x86/kernel/ftrace_64.S | 8 
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef CC_USING_FENTRY
 # define function_hook __fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
movl0x4(%ebp), %edx
subl$MCOUNT_INSN_SIZE, %eax
 
-   call*ftrace_trace_function
+   movlftrace_trace_function, %ecx
+   CALL_NOSPEC %ecx
 
popl%edx
popl%ecx
@@ -241,5 +243,5 @@ return_to_handler:
movl%eax, %ecx
popl%edx
popl%eax
-   jmp *%ecx
+   JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include 
 #include 
 #include 
-
+#include 
 
.code64
.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 * ip and parent ip are used and the list function is called when
 * function tracing is enabled.
 */
-   call   *ftrace_trace_function
-
+   movq ftrace_trace_function, %r8
+   CALL_NOSPEC %r8
restore_mcount_regs
 
jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 #endif

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  0672779c1f8e3ea1cc611bba50c53e224cbf4f31
Gitweb: https://git.kernel.org/tip/0672779c1f8e3ea1cc611bba50c53e224cbf4f31
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:15 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:12 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-d...@amazon.co.uk


---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  0672779c1f8e3ea1cc611bba50c53e224cbf4f31
Gitweb: https://git.kernel.org/tip/0672779c1f8e3ea1cc611bba50c53e224cbf4f31
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:15 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:12 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-d...@amazon.co.uk


---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9cc0128d93281e7f3be300b2b933d7c363b1ab26
Gitweb: https://git.kernel.org/tip/9cc0128d93281e7f3be300b2b933d7c363b1ab26
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:14 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:11 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Juergen Gross 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9cc0128d93281e7f3be300b2b933d7c363b1ab26
Gitweb: https://git.kernel.org/tip/9cc0128d93281e7f3be300b2b933d7c363b1ab26
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:14 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:11 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Juergen Gross 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  fe1678d070e36070c43005c71ce783df57852252
Gitweb: https://git.kernel.org/tip/fe1678d070e36070c43005c71ce783df57852252
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:09 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:09 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
integration becomes simple ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Cc: Tom Lendacky 
Link: 
https://lkml.kernel.org/r/1515508997-6154-4-git-send-email-d...@amazon.co.uk


---
 Documentation/admin-guide/kernel-parameters.txt |  28 +
 arch/x86/include/asm/nospec-branch.h|  10 ++
 arch/x86/kernel/cpu/bugs.c  | 158 +++-
 arch/x86/kernel/cpu/common.c|   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt 
b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
nosmt   [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
 
+   nospectre_v2[X86] Disable all mitigations for the Spectre variant 2
+   (indirect branch prediction) vulnerability. System may
+   allow data leaks with this option, which is equivalent
+   to spectre_v2=off.
+
noxsave [BUGS=X86] Disables x86 extended register state save
and restore using xsave. The kernel will fallback to
enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
sonypi.*=   [HW] Sony Programmable I/O Control Device driver
See Documentation/laptops/sonypi.txt
 
+   spectre_v2= [X86] Control mitigation of Spectre variant 2
+   (indirect branch speculation) vulnerability.
+
+   on   - unconditionally enable
+   off  - unconditionally disable
+   auto - kernel detects whether your CPU model is
+  vulnerable
+
+   Selecting 'on' will, and 'auto' may, choose a
+   mitigation method at run time according to the
+   CPU, the available microcode, the setting of the
+   CONFIG_RETPOLINE configuration option, and the
+   compiler with which the kernel was built.
+
+   Specific mitigations can also be selected manually:
+
+   retpoline - replace indirect branches
+   retpoline,generic - google's original retpoline
+   retpoline,amd - AMD-specific minimal thunk
+
+   Not specifying this option is equivalent to
+   spectre_v2=auto.
+
spia_io_base=   [HW,MTD]
spia_fio_base=
spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 7f58713..7d70ea9 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -105,5 +105,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+   SPECTRE_V2_NONE,
+   SPECTRE_V2_RETPOLINE_MINIMAL,
+   SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+   SPECTRE_V2_RETPOLINE_GENERIC,
+   SPECTRE_V2_RETPOLINE_AMD,
+   SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..4f7a2a0 100644
--- a/arch/x86/kernel/cpu/bugs.c

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  fe1678d070e36070c43005c71ce783df57852252
Gitweb: https://git.kernel.org/tip/fe1678d070e36070c43005c71ce783df57852252
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:09 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:09 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
integration becomes simple ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Cc: Tom Lendacky 
Link: 
https://lkml.kernel.org/r/1515508997-6154-4-git-send-email-d...@amazon.co.uk


---
 Documentation/admin-guide/kernel-parameters.txt |  28 +
 arch/x86/include/asm/nospec-branch.h|  10 ++
 arch/x86/kernel/cpu/bugs.c  | 158 +++-
 arch/x86/kernel/cpu/common.c|   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt 
b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
nosmt   [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
 
+   nospectre_v2[X86] Disable all mitigations for the Spectre variant 2
+   (indirect branch prediction) vulnerability. System may
+   allow data leaks with this option, which is equivalent
+   to spectre_v2=off.
+
noxsave [BUGS=X86] Disables x86 extended register state save
and restore using xsave. The kernel will fallback to
enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
sonypi.*=   [HW] Sony Programmable I/O Control Device driver
See Documentation/laptops/sonypi.txt
 
+   spectre_v2= [X86] Control mitigation of Spectre variant 2
+   (indirect branch speculation) vulnerability.
+
+   on   - unconditionally enable
+   off  - unconditionally disable
+   auto - kernel detects whether your CPU model is
+  vulnerable
+
+   Selecting 'on' will, and 'auto' may, choose a
+   mitigation method at run time according to the
+   CPU, the available microcode, the setting of the
+   CONFIG_RETPOLINE configuration option, and the
+   compiler with which the kernel was built.
+
+   Specific mitigations can also be selected manually:
+
+   retpoline - replace indirect branches
+   retpoline,generic - google's original retpoline
+   retpoline,amd - AMD-specific minimal thunk
+
+   Not specifying this option is equivalent to
+   spectre_v2=auto.
+
spia_io_base=   [HW,MTD]
spia_fio_base=
spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h 
b/arch/x86/include/asm/nospec-branch.h
index 7f58713..7d70ea9 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -105,5 +105,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+   SPECTRE_V2_NONE,
+   SPECTRE_V2_RETPOLINE_MINIMAL,
+   SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+   SPECTRE_V2_RETPOLINE_GENERIC,
+   SPECTRE_V2_RETPOLINE_AMD,
+   SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..4f7a2a0 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -11,6 +11,9 @@
 #include 
 #include 
 #include 
+
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -21,6 +24,8 @@
 #include 
 #include 
 
+static void __init spectre_v2_select_mitigation(void);
+
 void __init check_bugs(void)
 {
identify_boot_cpu();
@@ -30,6 +35,9 @@ void __init check_bugs(void)
  

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  713626ed94ad69d330fab550c94d7173cfcbf5ce
Gitweb: https://git.kernel.org/tip/713626ed94ad69d330fab550c94d7173cfcbf5ce
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:13 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:11 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-8-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/mshyperv.h | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
return U64_MAX;
 
__asm__ __volatile__("mov %4, %%r8\n"
-"call *%5"
+CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input_address)
-:  "r" (output_address), "m" (hv_hypercall_pg)
+:  "r" (output_address),
+   THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
if (!hv_hypercall_pg)
return U64_MAX;
 
-   __asm__ __volatile__("call *%7"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=A" (hv_status),
   "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 : "A" (control),
   "b" (input_address_hi),
   "D"(output_address_hi), "S"(output_address_lo),
-  "m" (hv_hypercall_pg)
+  THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory");
 #endif /* !x86_64 */
return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
 
 #ifdef CONFIG_X86_64
{
-   __asm__ __volatile__("call *%4"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input1)
-: "m" (hv_hypercall_pg)
+: THUNK_TARGET(hv_hypercall_pg)
 : "cc", "r8", "r9", "r10", "r11");
}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
 
-   __asm__ __volatile__ ("call *%5"
+   __asm__ __volatile__ (CALL_NOSPEC
  : "=A"(hv_status),
"+c"(input1_lo),
ASM_CALL_CONSTRAINT
  : "A" (control),
"b" (input1_hi),
-   "m" (hv_hypercall_pg)
+   THUNK_TARGET(hv_hypercall_pg)
  : "cc", "edi", "esi");
}
 #endif

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  713626ed94ad69d330fab550c94d7173cfcbf5ce
Gitweb: https://git.kernel.org/tip/713626ed94ad69d330fab550c94d7173cfcbf5ce
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:13 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:11 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-8-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/mshyperv.h | 18 ++
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
return U64_MAX;
 
__asm__ __volatile__("mov %4, %%r8\n"
-"call *%5"
+CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input_address)
-:  "r" (output_address), "m" (hv_hypercall_pg)
+:  "r" (output_address),
+   THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void 
*input, void *output)
if (!hv_hypercall_pg)
return U64_MAX;
 
-   __asm__ __volatile__("call *%7"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=A" (hv_status),
   "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 : "A" (control),
   "b" (input_address_hi),
   "D"(output_address_hi), "S"(output_address_lo),
-  "m" (hv_hypercall_pg)
+  THUNK_TARGET(hv_hypercall_pg)
 : "cc", "memory");
 #endif /* !x86_64 */
return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
 
 #ifdef CONFIG_X86_64
{
-   __asm__ __volatile__("call *%4"
+   __asm__ __volatile__(CALL_NOSPEC
 : "=a" (hv_status), ASM_CALL_CONSTRAINT,
   "+c" (control), "+d" (input1)
-: "m" (hv_hypercall_pg)
+: THUNK_TARGET(hv_hypercall_pg)
 : "cc", "r8", "r9", "r10", "r11");
}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 
input1)
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
 
-   __asm__ __volatile__ ("call *%5"
+   __asm__ __volatile__ (CALL_NOSPEC
  : "=A"(hv_status),
"+c"(input1_lo),
ASM_CALL_CONSTRAINT
  : "A" (control),
"b" (input1_hi),
-   "m" (hv_hypercall_pg)
+   THUNK_TARGET(hv_hypercall_pg)
  : "cc", "edi", "esi");
}
 #endif

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  8e8284e74cf80e1addf71d83794cba23f7a59e3a
Gitweb: https://git.kernel.org/tip/8e8284e74cf80e1addf71d83794cba23f7a59e3a
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:11 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:10 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Ingo Molnar 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-6-git-send-email-d...@amazon.co.uk


---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
/* kernel thread */
 1: movl%edi, %eax
-   call*%ebx
+   CALL_NOSPEC %ebx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
movl%ecx, %es
TRACE_IRQS_OFF
movl%esp, %eax  # pt_regs pointer
-   call*%edi
+   CALL_NOSPEC %edi
jmp ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 */
pushq   %rdi
movq$entry_SYSCALL_64_stage2, %rdi
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 * It might end up jumping to the slow path.  If it jumps, RAX
 * and all argument registers are clobbered.
 */
+#ifdef CONFIG_RETPOLINE
+   movqsys_call_table(, %rax, 8), %rax
+   call__x86_indirect_thunk_rax
+#else
call*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
movq%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
jmp entry_SYSCALL64_slow_path
 
 1:
-   jmp *%rax   /* Called from C */
+   JMP_NOSPEC %rax /* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
/* kernel thread */
movq%r12, %rdi
-   call*%rbx
+   CALL_NOSPEC %rbx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  8e8284e74cf80e1addf71d83794cba23f7a59e3a
Gitweb: https://git.kernel.org/tip/8e8284e74cf80e1addf71d83794cba23f7a59e3a
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:11 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:10 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Ingo Molnar 
Acked-by: Arjan van de Ven 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-6-git-send-email-d...@amazon.co.uk


---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
/* kernel thread */
 1: movl%edi, %eax
-   call*%ebx
+   CALL_NOSPEC %ebx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
movl%ecx, %es
TRACE_IRQS_OFF
movl%esp, %eax  # pt_regs pointer
-   call*%edi
+   CALL_NOSPEC %edi
jmp ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 */
pushq   %rdi
movq$entry_SYSCALL_64_stage2, %rdi
-   jmp *%rdi
+   JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 * It might end up jumping to the slow path.  If it jumps, RAX
 * and all argument registers are clobbered.
 */
+#ifdef CONFIG_RETPOLINE
+   movqsys_call_table(, %rax, 8), %rax
+   call__x86_indirect_thunk_rax
+#else
call*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
movq%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
jmp entry_SYSCALL64_slow_path
 
 1:
-   jmp *%rax   /* Called from C */
+   JMP_NOSPEC %rax /* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
/* kernel thread */
movq%r12, %rdi
-   call*%rbx
+   CALL_NOSPEC %rbx
/*
 * A kernel thread is allowed to return here after successfully
 * calling do_execve().  Exit to userspace to complete the execve()

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  b86d748af60a52fe5b2bfb28f4451f79e28d0b32
Gitweb: https://git.kernel.org/tip/b86d748af60a52fe5b2bfb28f4451f79e28d0b32
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:10 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:10 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-5-git-send-email-d...@amazon.co.uk


---
 arch/x86/crypto/aesni-intel_asm.S| 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S| 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S 
b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
pxor INC, STATE4
movdqu IV, 0x30(OUTP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x00(OUTP), INC
pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
_aesni_gf128mul_x_ble()
movups IV, (IVP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x40(OUTP), INC
pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
vpxor 14 * 16(%rax), %xmm15, %xmm14;
vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
vpxor 14 * 32(%rax), %ymm15, %ymm14;
vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S 
b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include 
 #include 
+#include 
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
movzxw  (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
-   jmp *bufp
+   JMP_NOSPEC bufp
 

## 2a) PROCESS FULL BLOCKS:

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  b86d748af60a52fe5b2bfb28f4451f79e28d0b32
Gitweb: https://git.kernel.org/tip/b86d748af60a52fe5b2bfb28f4451f79e28d0b32
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:10 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:10 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-5-git-send-email-d...@amazon.co.uk


---
 arch/x86/crypto/aesni-intel_asm.S| 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S| 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S 
b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
pxor INC, STATE4
movdqu IV, 0x30(OUTP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x00(OUTP), INC
pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
_aesni_gf128mul_x_ble()
movups IV, (IVP)
 
-   call *%r11
+   CALL_NOSPEC %r11
 
movdqu 0x40(OUTP), INC
pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
vpxor 14 * 16(%rax), %xmm15, %xmm14;
vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S 
b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include 
 #include 
+#include 
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
vpxor 14 * 32(%rax), %ymm15, %ymm14;
vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-   call *%r9;
+   CALL_NOSPEC %r9;
 
addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S 
b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include 
 #include 
+#include 
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
movzxw  (bufp, %rax, 2), len
lea crc_array(%rip), bufp
lea (bufp, len, 1), bufp
-   jmp *bufp
+   JMP_NOSPEC bufp
 

## 2a) PROCESS FULL BLOCKS:

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  3c0c41a7bc03932d02d7796b4735f9077eb2b320
Gitweb: https://git.kernel.org/tip/3c0c41a7bc03932d02d7796b4735f9077eb2b320
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:07 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:09 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-d...@amazon.co.uk


---
 arch/x86/Kconfig  |  13 
 arch/x86/Makefile |  10 
 arch/x86/include/asm/asm-prototypes.h |  25 
 arch/x86/include/asm/cpufeatures.h|   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++
 arch/x86/kernel/cpu/common.c  |   4 ++
 arch/x86/lib/Makefile |   1 +
 arch/x86/lib/retpoline.S  |  48 +++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
def_bool y
depends on X86_GOLDFISH
 
+config RETPOLINE
+   bool "Avoid speculative indirect branches in kernel"
+   default y
+   help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
 config INTEL_RDT
bool "Intel Resource Director Technology support"
default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+ifneq ($(RETPOLINE_CFLAGS),)
+KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+else
+$(warning CONFIG_RETPOLINE=y, but not supported by the compiler. 
Toolchain update recommended.)
+endif
+endif
+
 archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## 
reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## 
reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  3c0c41a7bc03932d02d7796b4735f9077eb2b320
Gitweb: https://git.kernel.org/tip/3c0c41a7bc03932d02d7796b4735f9077eb2b320
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:07 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 19:09:09 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-d...@amazon.co.uk


---
 arch/x86/Kconfig  |  13 
 arch/x86/Makefile |  10 
 arch/x86/include/asm/asm-prototypes.h |  25 
 arch/x86/include/asm/cpufeatures.h|   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++
 arch/x86/kernel/cpu/common.c  |   4 ++
 arch/x86/lib/Makefile |   1 +
 arch/x86/lib/retpoline.S  |  48 +++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
def_bool y
depends on X86_GOLDFISH
 
+config RETPOLINE
+   bool "Avoid speculative indirect branches in kernel"
+   default y
+   help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
 config INTEL_RDT
bool "Intel Resource Director Technology support"
default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern 
-mindirect-branch-register)
+ifneq ($(RETPOLINE_CFLAGS),)
+KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+else
+$(warning CONFIG_RETPOLINE=y, but not supported by the compiler. 
Toolchain update recommended.)
+endif
+endif
+
 archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h 
b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## 
reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## 
reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h 
b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- 

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  6a2b9eb3c189a848fbfc5b3a0b282bc03bc142af
Gitweb: https://git.kernel.org/tip/6a2b9eb3c189a848fbfc5b3a0b282bc03bc142af
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:15 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 18:28:24 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-d...@amazon.co.uk


---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  6a2b9eb3c189a848fbfc5b3a0b282bc03bc142af
Gitweb: https://git.kernel.org/tip/6a2b9eb3c189a848fbfc5b3a0b282bc03bc142af
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:15 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 18:28:24 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-d...@amazon.co.uk


---
 arch/x86/lib/checksum_32.S | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include 
 #include 
 #include 
-   
+#include 
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
negl %ebx
lea 45f(%ebx,%ebx,2), %ebx
testl %esi, %esi
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 
# Handle 2-byte-aligned regions
 20:addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
andl $-32,%edx
lea 3f(%ebx,%ebx), %ebx
testl %esi, %esi 
-   jmp *%ebx
+   JMP_NOSPEC %ebx
 1: addl $64,%esi
addl $64,%edi 
SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  5a6407e118225497fff1f503acbba96cd2abd21c
Gitweb: https://git.kernel.org/tip/5a6407e118225497fff1f503acbba96cd2abd21c
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:14 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 18:28:23 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Juergen Gross 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();
 

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  5a6407e118225497fff1f503acbba96cd2abd21c
Gitweb: https://git.kernel.org/tip/5a6407e118225497fff1f503acbba96cd2abd21c
Author: David Woodhouse 
AuthorDate: Tue, 9 Jan 2018 14:43:14 +
Committer:  Thomas Gleixner 
CommitDate: Wed, 10 Jan 2018 18:28:23 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse 
Signed-off-by: Thomas Gleixner 
Reviewed-by: Juergen Gross 
Acked-by: Arjan van de Ven 
Acked-by: Ingo Molnar 
Cc: gno...@lxorguk.ukuu.org.uk
Cc: Rik van Riel 
Cc: Andi Kleen 
Cc: Peter Zijlstra 
Cc: Linus Torvalds 
Cc: Jiri Kosina 
Cc: Andy Lutomirski 
Cc: Dave Hansen 
Cc: Kees Cook 
Cc: Tim Chen 
Cc: Greg Kroah-Hartman 
Cc: Paul Turner 
Link: 
https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-d...@amazon.co.uk


---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h 
b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
stac();
-   asm volatile("call *%[call]"
+   asm volatile(CALL_NOSPEC
 : __HYPERCALL_5PARAM
-: [call] "a" (_page[call])
+: [thunk_target] "a" (_page[call])
 : __HYPERCALL_CLOBBER5);
clac();