Currently RESTORE_CR3 does an unconditional flush
(SAVE_AND_SWITCH_TO_KERNEL_CR3 does not set bit 63 on \save_reg).
When restoring to a user ASID, check the user_asid_flush_mask to see
if we can avoid the flush.
For kernel ASIDs we can unconditionaly avoid the flush, since we do
explicit flushes for them.
Signed-off-by: Peter Zijlstra (Intel) <pet...@infradead.org>
---
arch/x86/entry/calling.h | 29 +++++++++++++++++++++++++++--
arch/x86/entry/entry_64.S | 4 ++--
2 files changed, 29 insertions(+), 4 deletions(-)
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -263,8 +263,33 @@ For 32-bit we have the following convent
.Ldone_\@:
.endm
-.macro RESTORE_CR3 save_reg:req
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
+
+ /* ASID bit 11 is for user */
+ bt $11, \save_reg
+ /*
+ * KERNEL pages can always resume with NOFLUSH as we do
+ * explicit flushes.
+ */
+ jnc .Lnoflush_\@
+
+ /*
+ * Check if there's a pending flush for the user ASID we're
+ * about to set.
+ */
+ movq \save_reg, \scratch_reg
+ andq $(0x7FF), \scratch_reg
+ bt \scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+ jnc .Lnoflush_\@
+
+ btr \scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+ jmp .Ldo_\@
+
+.Lnoflush_\@:
+ ALTERNATIVE "", "bts $63, \save_reg", X86_FEATURE_PCID
+
+.Ldo_\@:
/*
* The CR3 write could be avoided when not changing its value,
* but would require a CR3 read *and* a scratch register.
@@ -281,7 +306,7 @@ For 32-bit we have the following convent
.endm
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
.endm
-.macro RESTORE_CR3 save_reg:req
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
.endm
#endif
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1278,7 +1278,7 @@ ENTRY(paranoid_exit)
testl %ebx, %ebx /* swapgs needed? */
jnz .Lparanoid_exit_no_swapgs
TRACE_IRQS_IRETQ
- RESTORE_CR3 save_reg=%r14
+ RESTORE_CR3 scratch_reg=%rbx save_reg=%r14
SWAPGS_UNSAFE_STACK
jmp .Lparanoid_exit_restore
.Lparanoid_exit_no_swapgs:
@@ -1720,7 +1720,7 @@ ENTRY(nmi)
movq $-1, %rsi
call do_nmi
- RESTORE_CR3 save_reg=%r14
+ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
testl %ebx, %ebx /* swapgs needed? */
jnz nmi_restore
