When entering the kernel with CR3 pointing to the kernel's PGD, there's
no need to set it again. This will avoid a TLB flush on syscalls for tasks
running with the kernel's PGD (see next patch).
Signed-off-by: Willy Tarreau <w...@1wt.eu>
Cc: Andy Lutomirski <l...@kernel.org>
Cc: Borislav Petkov <b...@alien8.de>
Cc: Brian Gerst <brge...@gmail.com>
Cc: Dave Hansen <dave.han...@linux.intel.com>
Cc: Ingo Molnar <mi...@kernel.org>
Cc: Linus Torvalds <torva...@linux-foundation.org>
Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Thomas Gleixner <t...@linutronix.de>
Cc: Josh Poimboeuf <jpoim...@redhat.com>
Cc: "H. Peter Anvin" <h...@zytor.com>
Cc: Kees Cook <keesc...@chromium.org>
v2:
- updated comments according to Ingo's suggestions
- split the code to keep only the CR3 changes here
---
arch/x86/entry/calling.h | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 45a63e0..2c0d3b5 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -214,6 +214,11 @@
.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
mov %cr3, \scratch_reg
+
+ /* if we're already on the kernel PGD, we don't switch */
+ testq $(PTI_SWITCH_PGTABLES_MASK), \scratch_reg
+ jz .Lend_\@
+
ADJUST_KERNEL_CR3 \scratch_reg
mov \scratch_reg, %cr3
.Lend_\@:
@@ -262,6 +267,14 @@
ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI
movq %cr3, \scratch_reg
movq \scratch_reg, \save_reg
+
+ /*
+ * If we're already on the kernel PGD, we don't switch,
+ * we just save the current CR3.
+ */
+ testq $(PTI_SWITCH_PGTABLES_MASK), \scratch_reg
+ jz .Ldone_\@
+
/*
* Is the "switch mask" all zero? That means that both of
* these are zero:
@@ -284,6 +297,13 @@
.macro RESTORE_CR3 scratch_reg:req save_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
+ /*
+ * If we saved a kernel context on entry, we didn't switch the CR3,
+ * so we don't need to restore it on the way out either:
+ */
+ testq $(PTI_SWITCH_PGTABLES_MASK), \save_reg
+ jz .Lend_\@
+
ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
/*
--
1.7.12.1
