In commit:
a7295fd53c39 ("x86/mm/cpa: Use flush_tlb_kernel_range()")
I misread the cpa array code and incorrectly used
tlb_flush_kernel_range(), resulting in missing TLB flushes and
consequent failures.
Iterate the array and invalidate the individual pages instead.
Fixes: a7295fd53c39 ("x86/mm/cpa: Use flush_tlb_kernel_range()")
Reported-by: "StDenis, Tom" <[email protected]>
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
---
arch/x86/mm/mm_internal.h | 2 +
arch/x86/mm/pageattr.c | 62 ++++++++++++++++++++++++++++------------------
arch/x86/mm/tlb.c | 4 ++
3 files changed, 43 insertions(+), 25 deletions(-)
--- a/arch/x86/mm/mm_internal.h
+++ b/arch/x86/mm/mm_internal.h
@@ -19,4 +19,6 @@ extern int after_bootmem;
void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache);
+extern unsigned long tlb_single_page_flush_ceiling;
+
#endif /* __X86_MM_INTERNAL_H */
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -26,6 +26,8 @@
#include <asm/pat.h>
#include <asm/set_memory.h>
+#include "mm_internal.h"
+
/*
* The current flushing context - we pass it instead of 5 arguments:
*/
@@ -302,20 +304,16 @@ static void cpa_flush_all(unsigned long
on_each_cpu(__cpa_flush_all, (void *) cache, 1);
}
-static bool __cpa_flush_range(unsigned long start, int numpages, int cache)
+static bool cpa_check_flush_all(int cache)
{
BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
- WARN_ON(PAGE_ALIGN(start) != start);
-
if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
cpa_flush_all(cache);
return true;
}
- flush_tlb_kernel_range(start, start + PAGE_SIZE * numpages);
-
- return !cache;
+ return false;
}
static void cpa_flush_range(unsigned long start, int numpages, int cache)
@@ -323,7 +321,14 @@ static void cpa_flush_range(unsigned lon
unsigned int i, level;
unsigned long addr;
- if (__cpa_flush_range(start, numpages, cache))
+ WARN_ON(PAGE_ALIGN(start) != start);
+
+ if (cpa_check_flush_all(cache))
+ return;
+
+ flush_tlb_kernel_range(start, start + PAGE_SIZE * numpages);
+
+ if (!cache)
return;
/*
@@ -343,13 +348,28 @@ static void cpa_flush_range(unsigned lon
}
}
-static void cpa_flush_array(unsigned long baddr, unsigned long *start,
- int numpages, int cache,
- int in_flags, struct page **pages)
+void __cpa_flush_array(void *data)
{
- unsigned int i, level;
+ struct cpa_data *cpa = data;
+ unsigned int i;
- if (__cpa_flush_range(baddr, numpages, cache))
+ for (i = 0; i < cpa->numpages; i++)
+ __flush_tlb_one_kernel(__cpa_addr(cpa, i));
+}
+
+static void cpa_flush_array(struct cpa_data *cpa, int cache)
+{
+ unsigned int i;
+
+ if (cpa_check_flush_all(cache))
+ return;
+
+ if (cpa->numpages <= tlb_single_page_flush_ceiling)
+ on_each_cpu(__cpa_flush_array, cpa, 1);
+ else
+ flush_tlb_all();
+
+ if (!cache)
return;
/*
@@ -358,15 +378,11 @@ static void cpa_flush_array(unsigned lon
* will cause all other CPUs to flush the same
* cachelines:
*/
- for (i = 0; i < numpages; i++) {
- unsigned long addr;
+ for (i = 0; i < cpa->numpages; i++) {
+ unsigned long addr = __cpa_addr(cpa, i);
+ unsigned int level;
pte_t *pte;
- if (in_flags & CPA_PAGES_ARRAY)
- addr = (unsigned long)page_address(pages[i]);
- else
- addr = start[i];
-
pte = lookup_address(addr, &level);
/*
@@ -1765,12 +1781,10 @@ static int change_page_attr_set_clr(unsi
goto out;
}
- if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
- cpa_flush_array(baddr, addr, numpages, cache,
- cpa.flags, pages);
- } else {
+ if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
+ cpa_flush_array(&cpa, cache);
+ else
cpa_flush_range(baddr, numpages, cache);
- }
out:
return ret;
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -16,6 +16,8 @@
#include <asm/apic.h>
#include <asm/uv/uv.h>
+#include "mm_internal.h"
+
/*
* TLB flushing, formerly SMP-only
* c/o Linus Torvalds.
@@ -664,7 +666,7 @@ void native_flush_tlb_others(const struc
*
* This is in units of pages.
*/
-static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
+unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long end, unsigned int stride_shift,
