From: Nicholas Piggin <npig...@gmail.com> When platform doesn't support GTSE, let TLB invalidation requests for radix guests be off-loaded to the host using H_RPT_INVALIDATE hcall
Signed-off-by: Nicholas Piggin <npig...@gmail.com> Signed-off-by: Bharata B Rao <bhar...@linux.ibm.com> --- arch/powerpc/include/asm/hvcall.h | 1 + arch/powerpc/include/asm/plpar_wrappers.h | 14 +++ arch/powerpc/mm/book3s64/radix_tlb.c | 105 ++++++++++++++++++++-- 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index e90c073e437e..08917147415b 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -335,6 +335,7 @@ #define H_GET_24X7_CATALOG_PAGE 0xF078 #define H_GET_24X7_DATA 0xF07C #define H_GET_PERF_COUNTER_INFO 0xF080 +#define H_RPT_INVALIDATE 0xF084 /* Platform-specific hcalls used for nested HV KVM */ #define H_SET_PARTITION_TABLE 0xF800 diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h index 4497c8afb573..e952139b0e47 100644 --- a/arch/powerpc/include/asm/plpar_wrappers.h +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -334,6 +334,13 @@ static inline long plpar_get_cpu_characteristics(struct h_cpu_char_result *p) return rc; } +static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 what, + u64 pages, u64 start, u64 end) +{ + return plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, what, + pages, start, end); +} + #else /* !CONFIG_PPC_PSERIES */ static inline long plpar_set_ciabr(unsigned long ciabr) @@ -346,6 +353,13 @@ static inline long plpar_pte_read_4(unsigned long flags, unsigned long ptex, { return 0; } + +static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 what, + u64 pages, u64 start, u64 end) +{ + return 0; +} + #endif /* CONFIG_PPC_PSERIES */ #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */ diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index b5cc9b23cf02..4dd1d3c75562 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -16,11 +16,39 @@ #include <asm/tlbflush.h> #include <asm/trace.h> #include <asm/cputhreads.h> +#include <asm/plpar_wrappers.h> #define RIC_FLUSH_TLB 0 #define RIC_FLUSH_PWC 1 #define RIC_FLUSH_ALL 2 +#define H_TLBI_TLB 0x0001 +#define H_TLBI_PWC 0x0002 +#define H_TLBI_PRS 0x0004 + +#define H_TLBI_TARGET_CMMU 0x01 +#define H_TLBI_TARGET_CMMU_LOCAL 0x02 +#define H_TLBI_TARGET_NMMU 0x04 + +#define H_TLBI_PAGE_ALL (-1UL) +#define H_TLBI_PAGE_4K 0x01 +#define H_TLBI_PAGE_64K 0x02 +#define H_TLBI_PAGE_2M 0x04 +#define H_TLBI_PAGE_1G 0x08 + +static inline u64 psize_to_h_tlbi(unsigned long psize) +{ + if (psize == MMU_PAGE_4K) + return H_TLBI_PAGE_4K; + if (psize == MMU_PAGE_64K) + return H_TLBI_PAGE_64K; + if (psize == MMU_PAGE_2M) + return H_TLBI_PAGE_2M; + if (psize == MMU_PAGE_1G) + return H_TLBI_PAGE_1G; + return H_TLBI_PAGE_ALL; +} + /* * tlbiel instruction for radix, set invalidation * i.e., r=1 and is=01 or is=10 or is=11 @@ -694,7 +722,14 @@ void radix__flush_tlb_mm(struct mm_struct *mm) goto local; } - if (cputlb_use_tlbie()) { + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long targ = H_TLBI_TARGET_CMMU; + + if (atomic_read(&mm->context.copros) > 0) + targ |= H_TLBI_TARGET_NMMU; + pseries_rpt_invalidate(pid, targ, H_TLBI_TLB, + H_TLBI_PAGE_ALL, 0, -1UL); + } else if (cputlb_use_tlbie()) { if (mm_needs_flush_escalation(mm)) _tlbie_pid(pid, RIC_FLUSH_ALL); else @@ -727,7 +762,16 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) goto local; } } - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long targ = H_TLBI_TARGET_CMMU; + unsigned long what = H_TLBI_TLB | H_TLBI_PWC | + H_TLBI_PRS; + + if (atomic_read(&mm->context.copros) > 0) + targ |= H_TLBI_TARGET_NMMU; + pseries_rpt_invalidate(pid, targ, what, + H_TLBI_PAGE_ALL, 0, -1UL); + } else if (cputlb_use_tlbie()) _tlbie_pid(pid, RIC_FLUSH_ALL); else _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); @@ -760,7 +804,17 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, exit_flush_lazy_tlbs(mm); goto local; } - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long targ = H_TLBI_TARGET_CMMU; + unsigned long pages = psize_to_h_tlbi(psize); + unsigned long page_size = + 1UL << mmu_psize_to_shift(psize); + + if (atomic_read(&mm->context.copros) > 0) + targ |= H_TLBI_TARGET_NMMU; + pseries_rpt_invalidate(pid, targ, H_TLBI_TLB, pages, + vmaddr, vmaddr + page_size); + } else if (cputlb_use_tlbie()) _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); else _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); @@ -810,7 +864,13 @@ static inline void _tlbiel_kernel_broadcast(void) */ void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) { - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long targ = H_TLBI_TARGET_CMMU | H_TLBI_TARGET_NMMU; + unsigned long what = H_TLBI_TLB | H_TLBI_PWC | H_TLBI_PRS; + + pseries_rpt_invalidate(0, targ, what, H_TLBI_PAGE_ALL, + start, end); + } else if (cputlb_use_tlbie()) _tlbie_pid(0, RIC_FLUSH_ALL); else _tlbiel_kernel_broadcast(); @@ -864,7 +924,17 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, nr_pages > tlb_local_single_page_flush_ceiling); } - if (full) { + if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + unsigned long targ = H_TLBI_TARGET_CMMU; + unsigned long pages = psize_to_h_tlbi(mmu_virtual_psize); + + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) + pages |= psize_to_h_tlbi(MMU_PAGE_2M); + if (atomic_read(&mm->context.copros) > 0) + targ |= H_TLBI_TARGET_NMMU; + pseries_rpt_invalidate(pid, targ, H_TLBI_TLB, pages, + start, end); + } else if (full) { if (local) { _tlbiel_pid(pid, RIC_FLUSH_TLB); } else { @@ -1046,7 +1116,17 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, nr_pages > tlb_local_single_page_flush_ceiling); } - if (full) { + if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + unsigned long targ = H_TLBI_TARGET_CMMU; + unsigned long what = H_TLBI_TLB; + unsigned long pages = psize_to_h_tlbi(psize); + + if (also_pwc) + what |= H_TLBI_PWC; + if (atomic_read(&mm->context.copros) > 0) + targ |= H_TLBI_TARGET_NMMU; + pseries_rpt_invalidate(pid, targ, what, pages, start, end); + } else if (full) { if (local) { _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); } else { @@ -1111,7 +1191,18 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) exit_flush_lazy_tlbs(mm); goto local; } - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long targ = H_TLBI_TARGET_CMMU; + unsigned long what = H_TLBI_TLB | H_TLBI_PWC | + H_TLBI_PRS; + unsigned long pages = + psize_to_h_tlbi(mmu_virtual_psize); + + if (atomic_read(&mm->context.copros) > 0) + targ |= H_TLBI_TARGET_NMMU; + pseries_rpt_invalidate(pid, targ, what, pages, + addr, end); + } else if (cputlb_use_tlbie()) _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); else _tlbiel_va_range_multicast(mm, -- 2.21.3