On Mon, Nov 27, 2017 at 09:16:19PM -0800, Andy Lutomirski wrote:
> We need to split up __flush_tlb_one() into __flush_tlb_one() and
> __flush_tlb_one_kernel(). We've gotten away with having a single
> function for both this long because we've never had PCID on and
> nonglobal kernel mappings around. So we're busted starting with
> "x86/mm/kaiser: Disable global pages by default with KAISER", which
> means that we have a potential corruption issue affecting anyone who
> tries to bisect the series.
Didn't do that..
> Then we need to make the kernel variant do something sane (presumably
> just call __flush_tlb_all if we have PCID && !PGE). And, for the user
> variant, we need a straightforward, clean, efficient way to mark a
> given address space on a given CPU as needing a usermode PCID flush
> when its usermode tables are next loaded. This patch isn't it.
Did give this a try, mostly also to get PCID + !INVPCID working for my
IVB.
The below is fairly ugly but it does boot and build a kernel so its not
immensely broken.
---
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 07fa7fdd7b68..d7f1be6ccc97 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -4,6 +4,7 @@
#include <asm/cpufeatures.h>
#include <asm/page_types.h>
#include <asm/pgtable_types.h>
+#include <asm/percpu.h>
/*
@@ -220,7 +221,21 @@ For 32-bit we have the following conventions - kernel is
built with
.macro SWITCH_TO_USER_CR3 scratch_reg:req
STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
mov %cr3, \scratch_reg
- ADJUST_USER_CR3 \scratch_reg
+ push \scratch_reg
+ andq $(0x7FF), \scratch_reg
+ bt \scratch_reg, PER_CPU_VAR(__asid_flush)
+ jnc .Lnoflush_\@
+
+ btr \scratch_reg, PER_CPU_VAR(__asid_flush)
+ pop \scratch_reg
+ jmp .Ldo_\@
+
+.Lnoflush_\@:
+ pop \scratch_reg
+ ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID
+
+.Ldo_\@:
+ orq $(KAISER_SWITCH_MASK), \scratch_reg
mov \scratch_reg, %cr3
.Lend_\@:
.endm
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 27eb7e8c5e84..1fb137da4c9f 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -9,6 +9,7 @@
#include <asm/cpufeature.h>
#include <asm/special_insns.h>
#include <asm/smp.h>
+#include <asm/kaiser.h>
static inline void __invpcid(unsigned long pcid, unsigned long addr,
unsigned long type)
@@ -347,9 +348,33 @@ static inline void cr4_set_bits_and_update_boot(unsigned
long mask)
extern void initialize_tlbstate_and_flush(void);
+DECLARE_PER_CPU(unsigned long, __asid_flush);
+
+/*
+ * Given an asid, flush the corresponding KAISER user ASID.
+ */
+static inline void flush_user_asid(u16 asid)
+{
+ /* There is no user ASID if KAISER is off */
+ if (!IS_ENABLED(CONFIG_KAISER))
+ return;
+ /*
+ * We only have a single ASID if PCID is off and the CR3
+ * write will have flushed it.
+ */
+ if (!cpu_feature_enabled(X86_FEATURE_PCID))
+ return;
+
+ if (!kaiser_enabled)
+ return;
+
+ __set_bit(kern_asid(asid), this_cpu_ptr(&__asid_flush));
+}
+
static inline void __native_flush_tlb(void)
{
if (!cpu_feature_enabled(X86_FEATURE_INVPCID)) {
+#if 0
/*
* native_write_cr3() only clears the current PCID if
* CR4 has X86_CR4_PCIDE set. In other words, this does
@@ -358,9 +383,10 @@ static inline void __native_flush_tlb(void)
* With KAISER and PCIDs, the means that we did not
* flush the user PCID. Warn if it gets called.
*/
- if (IS_ENABLED(CONFIG_KAISER))
- WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) &
- X86_CR4_PCIDE);
+ if (IS_ENABLED(CONFIG_KAISER) && kaiser_enabled)
+ WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) &
X86_CR4_PCIDE);
+#endif
+ flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
/*
* If current->mm == NULL then we borrow a mm
* which may change during a task switch and
@@ -435,6 +461,8 @@ static inline void __native_flush_tlb_single(unsigned long
addr)
* early.
*/
if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) {
+ flush_user_asid(loaded_mm_asid);
+
asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
return;
}
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index 72f115178d14..2dcd01615772 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -218,11 +218,13 @@ static void setup_pcid(void)
* INVPCID. Just avoid using PCIDs at all if we
* have KAISER and do not have INVPCID.
*/
+#if 0
if (!IS_ENABLED(CONFIG_X86_GLOBAL_PAGES) &&
- !boot_cpu_has(X86_FEATURE_INVPCID)) {
+ kaiser_enabled && !boot_cpu_has(X86_FEATURE_INVPCID)) {
setup_clear_cpu_cap(X86_FEATURE_PCID);
return;
}
+#endif
/*
* This can't be cr4_set_bits_and_update_boot() --
* the trampoline code can't handle CR4.PCIDE and
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index f75b6eb47a6d..4ed1d0dfd54f 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -100,55 +100,14 @@ static void choose_new_asid(struct mm_struct *next, u64
next_tlb_gen,
*need_flush = true;
}
-/*
- * Given a kernel asid, flush the corresponding KAISER
- * user ASID.
- */
-static void flush_user_asid(pgd_t *pgd, u16 kern_asid)
-{
- /* There is no user ASID if KAISER is off */
- if (!IS_ENABLED(CONFIG_KAISER))
- return;
- /*
- * We only have a single ASID if PCID is off and the CR3
- * write will have flushed it.
- */
- if (!cpu_feature_enabled(X86_FEATURE_PCID))
- return;
- /*
- * With PCIDs enabled, write_cr3() only flushes TLB
- * entries for the current (kernel) ASID. This leaves
- * old TLB entries for the user ASID in place and we must
- * flush that context separately. We can theoretically
- * delay doing this until we actually load up the
- * userspace CR3, but do it here for simplicity.
- */
- if (cpu_feature_enabled(X86_FEATURE_INVPCID)) {
- invpcid_flush_single_context(user_asid(kern_asid));
- } else {
- /*
- * On systems with PCIDs, but no INVPCID, the only
- * way to flush a PCID is a CR3 write. Note that
- * we use the kernel page tables with the *user*
- * ASID here.
- */
- unsigned long user_asid_flush_cr3;
- user_asid_flush_cr3 = build_cr3(pgd, user_asid(kern_asid));
- write_cr3(user_asid_flush_cr3);
- /*
- * We do not use PCIDs with KAISER unless we also
- * have INVPCID. Getting here is unexpected.
- */
- WARN_ON_ONCE(1);
- }
-}
+__visible DEFINE_PER_CPU(unsigned long, __asid_flush);
static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
{
unsigned long new_mm_cr3;
if (need_flush) {
- flush_user_asid(pgdir, new_asid);
+ flush_user_asid(new_asid);
new_mm_cr3 = build_cr3(pgdir, new_asid);
} else {
new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);
