Instead of relying on INVPCID to shoot down user PCID, delay the
invalidate until we switch to the user page-tables.
This gets rid of the INVPCID dependence for KAISER PCID.
XXX we could do a much larger ALTERNATIVE, there is no point in
testing the mask if we don't have PCID support.
Suggested-by: Andy Lutomirski <l...@amacapital.net>
Signed-off-by: Peter Zijlstra (Intel) <pet...@infradead.org>
---
arch/x86/entry/calling.h | 29 +++++++++++++++++------
arch/x86/include/asm/tlbflush.h | 39 +++++++++++++++++++++++--------
arch/x86/mm/init.c | 13 ----------
arch/x86/mm/tlb.c | 49 +---------------------------------------
4 files changed, 53 insertions(+), 77 deletions(-)
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -4,6 +4,7 @@
#include <asm/cpufeatures.h>
#include <asm/page_types.h>
#include <asm/pgtable_types.h>
+#include <asm/percpu.h>
/*
@@ -203,12 +204,6 @@ For 32-bit we have the following convent
andq $(~KAISER_SWITCH_MASK), \reg
.endm
-.macro ADJUST_USER_CR3 reg:req
- ALTERNATIVE "", "bts $63, \reg", X86_FEATURE_PCID
- /* Set user PCID bit, and move CR3 up a page to the user page tables: */
- orq $(KAISER_SWITCH_MASK), \reg
-.endm
-
.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
mov %cr3, \scratch_reg
@@ -220,7 +215,27 @@ For 32-bit we have the following convent
.macro SWITCH_TO_USER_CR3 scratch_reg:req
STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
mov %cr3, \scratch_reg
- ADJUST_USER_CR3 \scratch_reg
+
+ /*
+ * Test if the ASID needs a flush.
+ */
+ push \scratch_reg /* preserve CR3 */
+ andq $(0x7FF), \scratch_reg /* mask ASID */
+ bt \scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+ jnc .Lnoflush_\@
+
+ /* Flush needed, clear the bit */
+ btr \scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+ pop \scratch_reg /* original CR3 */
+ jmp .Ldo_\@
+
+.Lnoflush_\@:
+ pop \scratch_reg /* original CR3 */
+ ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID
+
+.Ldo_\@:
+ /* Flip the PGD and ASID to the user version */
+ orq $(KAISER_SWITCH_MASK), \scratch_reg
mov \scratch_reg, %cr3
.Lend_\@:
.endm
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -348,19 +348,37 @@ static inline void cr4_set_bits_and_upda
extern void initialize_tlbstate_and_flush(void);
+DECLARE_PER_CPU(unsigned long, user_asid_flush_mask);
+
+/*
+ * Given an ASID, flush the corresponding user ASID.
+ * We can delay this until the next time we switch to it.
+ *
+ * See SWITCH_TO_USER_CR3.
+ */
+static inline void flush_user_asid(u16 asid)
+{
+ /* There is no user ASID if KAISER is off */
+ if (!IS_ENABLED(CONFIG_KAISER))
+ return;
+
+ /*
+ * We only have a single ASID if PCID is off and the CR3
+ * write will have flushed it.
+ */
+ if (!cpu_feature_enabled(X86_FEATURE_PCID))
+ return;
+
+ if (!kaiser_enabled)
+ return;
+
+ __set_bit(kern_asid(asid), this_cpu_ptr(&user_asid_flush_mask));
+}
+
static inline void __native_flush_tlb(void)
{
if (!cpu_feature_enabled(X86_FEATURE_INVPCID)) {
- /*
- * native_write_cr3() only clears the current PCID if
- * CR4 has X86_CR4_PCIDE set. In other words, this does
- * not fully flush the TLB if PCIDs are in use.
- *
- * With KAISER and PCIDs, that means that we did not
- * flush the user PCID. Warn if it gets called.
- */
- if (IS_ENABLED(CONFIG_KAISER) && kaiser_enabled)
- WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) &
X86_CR4_PCIDE);
+ flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
/*
* If current->mm == NULL then we borrow a mm
@@ -436,6 +454,7 @@ static inline void __native_flush_tlb_si
* early.
*/
if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) {
+ flush_user_asid(loaded_mm_asid);
asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
return;
}
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -211,19 +211,6 @@ static void setup_pcid(void)
if (boot_cpu_has(X86_FEATURE_PGE)) {
/*
- * KAISER uses a PCID for the kernel and another
- * for userspace. Both PCIDs need to be flushed
- * when the TLB flush functions are called. But,
- * flushing *another* PCID is insane without
- * INVPCID. Just avoid using PCIDs at all if we
- * have KAISER and do not have INVPCID.
- */
- if (!IS_ENABLED(CONFIG_X86_GLOBAL_PAGES) &&
- kaiser_enabled && !boot_cpu_has(X86_FEATURE_INVPCID)) {
- setup_clear_cpu_cap(X86_FEATURE_PCID);
- return;
- }
- /*
* This can't be cr4_set_bits_and_update_boot() --
* the trampoline code can't handle CR4.PCIDE and
* it wouldn't do any good anyway. Despite the name,
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -101,59 +101,14 @@ static void choose_new_asid(struct mm_st
*need_flush = true;
}
-/*
- * Given a kernel asid, flush the corresponding KAISER
- * user ASID.
- */
-static void flush_user_asid(pgd_t *pgd, u16 kern_asid)
-{
- /* There is no user ASID if KAISER is off */
- if (!IS_ENABLED(CONFIG_KAISER))
- return;
- /*
- * We only have a single ASID if PCID is off and the CR3
- * write will have flushed it.
- */
- if (!cpu_feature_enabled(X86_FEATURE_PCID))
- return;
-
- if (!kaiser_enabled)
- return;
-
- /*
- * With PCIDs enabled, write_cr3() only flushes TLB
- * entries for the current (kernel) ASID. This leaves
- * old TLB entries for the user ASID in place and we must
- * flush that context separately. We can theoretically
- * delay doing this until we actually load up the
- * userspace CR3, but do it here for simplicity.
- */
- if (cpu_feature_enabled(X86_FEATURE_INVPCID)) {
- invpcid_flush_single_context(user_asid(kern_asid));
- } else {
- /*
- * On systems with PCIDs, but no INVPCID, the only
- * way to flush a PCID is a CR3 write. Note that
- * we use the kernel page tables with the *user*
- * ASID here.
- */
- unsigned long user_asid_flush_cr3;
- user_asid_flush_cr3 = build_cr3(pgd, user_asid(kern_asid));
- write_cr3(user_asid_flush_cr3);
- /*
- * We do not use PCIDs with KAISER unless we also
- * have INVPCID. Getting here is unexpected.
- */
- WARN_ON_ONCE(1);
- }
-}
+__visible DEFINE_PER_CPU(unsigned long, user_asid_flush_mask);
static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
{
unsigned long new_mm_cr3;
if (need_flush) {
- flush_user_asid(pgdir, new_asid);
+ flush_user_asid(new_asid);
new_mm_cr3 = build_cr3(pgdir, new_asid);
} else {
new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);
