From: Dave Hansen <dave.han...@linux.intel.com>
From: David Woodhouse <d...@amazon.co.uk>
retpoline is a special sequence on Intel CPUs to stop speculation for
indirect branches.
Provide assembler infrastructure to use retpoline by the compiler
and for assembler. We add the out of line trampoline used by the
compiler, and NOSPEC_JUMP / NOSPEC_CALL macros for assembler
[Originally from David and Tim, heavily hacked by AK]
Signed-off-by: David Woodhouse <d...@amazon.co.uk>
Signed-off-by: Tim Chen <tim.c.c...@linux.intel.com>
Signed-off-by: Andi Kleen <a...@linux.intel.com>
---
arch/x86/include/asm/jump-asm.h | 47 +++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/vmlinux.lds.S | 1 +
arch/x86/lib/Makefile | 1 +
arch/x86/lib/retpoline.S | 35 ++++++++++++++++++++++++++++++
4 files changed, 84 insertions(+)
create mode 100644 arch/x86/include/asm/jump-asm.h
create mode 100644 arch/x86/lib/retpoline.S
diff --git a/arch/x86/include/asm/jump-asm.h b/arch/x86/include/asm/jump-asm.h
new file mode 100644
index 000000000000..953c391991b9
--- /dev/null
+++ b/arch/x86/include/asm/jump-asm.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef JUMP_ASM_H
+#define JUMP_ASM_H 1
+
+#ifdef __ASSEMBLY__
+
+/*
+ * Jump to an indirect pointer without speculation.
+ *
+ * The out of line __x86.indirect_thunk has special code sequences
+ * to stop speculation.
+ */
+
+.macro NOSPEC_JMP target
+ push \target
+ jmp __x86.indirect_thunk
+.endm
+
+
+/*
+ * Call an indirect pointer without speculation.
+ */
+
+.macro NOSPEC_CALL target
+ jmp 1221f
+1222:
+ push \target
+ jmp __x86.indirect_thunk
+1221:
+ call 1222b
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#define NOSPEC_JUMP(t) \
+ "push " t "; " \
+ "jmp __x86.indirect_thunk; "
+
+#define NOSPEC_CALL(t) \
+ " jmp 1221f; " \
+ "1222: push " t ";" \
+ " jmp __x86.indirect_thunk;" \
+ "1221: call 1222b;"
+
+#endif /* !__ASSEMBLY */
+
+#endif
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 1e413a9326aa..2e64241a6664 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -103,6 +103,7 @@ SECTIONS
/* bootstrapping code */
HEAD_TEXT
. = ALIGN(8);
+ *(.text.__x86.indirect_thunk)
TEXT_TEXT
SCHED_TEXT
CPUIDLE_TEXT
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 7b181b61170e..ec7a329b9b3c 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-y += retpoline.o
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 000000000000..cb40781adbfe
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Out of line jump trampoline for calls that disable speculation.
+ *
+ * This is a special sequence that prevents the CPU speculating
+ * for indirect calls.
+ *
+ * This can be called by gcc generated code, or with the asm macros
+ * in asm/jump-asm.h
+ */
+
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/export.h>
+
+ .section .text.__x86.indirect_thunk,"ax"
+
+ENTRY(__x86.indirect_thunk)
+ CFI_STARTPROC
+ call retpoline_call_target
+2:
+ lfence /* stop speculation */
+ jmp 2b
+retpoline_call_target:
+#ifdef CONFIG_64BIT
+ lea 8(%rsp), %rsp
+#else
+ lea 4(%esp), %esp
+#endif
+ ret
+ CFI_ENDPROC
+ENDPROC(__x86.indirect_thunk)
+
+ EXPORT_SYMBOL(__x86.indirect_thunk)
--
2.14.3
