memfd_secret() keeps a secret by taking its page out of the kernel linear map. The process that owns the page can still read and write it through its own mapping, but the kernel no longer has a translation for that physical page, so a kernel read or a speculative walk of the linear map cannot see it.
Note that set_memory_np() and set_memory_p() already clear and restore the present bit for debug_pagealloc and KFENCE, so the same direct-map helpers are used here to implement ARCH_HAS_SET_DIRECT_MAP for radix. Note that they can only run when the linear map is already made of base pages. For implementing this feature when the linear map is mapped with 2M (PMD) or 1G (PUD) mapping - a 2M or 1G leaf would have to be broken into PTEs first and doing that while the large translation is still in the TLB installs a second translation for the same address at a different page size. On this, the processor architecture manual says that it is fatal: If a given virtual address is mapped by two overlapping pages of different sizes (for example, a small page is mapped initially but is later re-mapped into a larger page without invalidating the original TLB entry), a machine check interrupt can result with an indication that either a parity error or a multi-hit occurred when the TLB was accessed to translate the address. The error condition can be corrected by invalidating the entire TLB and SLB. The same multi-hit applies if the large page was installed first and is then replaced by PTEs without invalidating that TLB entry. Since today there is no support for doing that split safely, so can_set_direct_map() stays false unless the linear map is already page-sized. Signed-off-by: Ritesh Harjani (IBM) <[email protected]> --- arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/set_memory.h | 9 +++ arch/powerpc/mm/pageattr.c | 96 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 2580e27e4328..3c5c07c67a5b 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -150,6 +150,7 @@ config PPC select ARCH_HAS_PTDUMP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64 + select ARCH_HAS_SET_DIRECT_MAP if PPC_RADIX_MMU select ARCH_HAS_SET_MEMORY select ARCH_HAS_STRICT_KERNEL_RWX if (PPC_BOOK3S || PPC_8xx) && !HIBERNATION select ARCH_HAS_STRICT_KERNEL_RWX if PPC_85xx && !HIBERNATION && !RANDOMIZE_BASE diff --git a/arch/powerpc/include/asm/set_memory.h b/arch/powerpc/include/asm/set_memory.h index 9c8d5747755d..24779c00c1ca 100644 --- a/arch/powerpc/include/asm/set_memory.h +++ b/arch/powerpc/include/asm/set_memory.h @@ -48,4 +48,13 @@ static inline int __must_check set_memory_rox(unsigned long addr, int numpages) } #define set_memory_rox set_memory_rox +#ifdef CONFIG_ARCH_HAS_SET_DIRECT_MAP +bool can_set_direct_map(void); +#define can_set_direct_map can_set_direct_map + +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); +int set_direct_map_default_noflush(struct page *page, unsigned int numpages); +bool kernel_page_present(struct page *page); +#endif + #endif diff --git a/arch/powerpc/mm/pageattr.c b/arch/powerpc/mm/pageattr.c index ac22bf28086f..a85ac2419e6d 100644 --- a/arch/powerpc/mm/pageattr.c +++ b/arch/powerpc/mm/pageattr.c @@ -103,6 +103,102 @@ int change_memory_attr(unsigned long addr, int numpages, long action) change_page_attr, (void *)action); } +#ifdef CONFIG_ARCH_HAS_SET_DIRECT_MAP +/* + * Removing a single page from the linear mapping requires that the linear + * mapping is built out of PAGE_SIZE mappings in the first place. Radix only + * does that when explicitly asked to and there is no support for splitting + * a live 2M/1G mapping yet. + * TODO: Hash is also not supported yet. + */ +bool can_set_direct_map(void) +{ + if (!radix_enabled()) + return false; + + return linear_map_uses_base_pagesize(); +} + +/* + * Note that the set_memory_p()/set_memory_np() helpers below already flush the + * TLB for the range they touch, so the _noflush() naming is not honoured + * literally here. + * Since as of now this is not in the performance path, so it should be ok. + * Later, once the support for breaking the large mappings (PUD/PMD) into PTE is + * added, then this can be improved since it will anyway require a custom + * implementation for walking the page table and breaking the mapping. + */ +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages) +{ + unsigned long addr = (unsigned long)page_address(page); + + if (!can_set_direct_map()) + return 0; + + return set_memory_np(addr, numpages); +} + +int set_direct_map_default_noflush(struct page *page, unsigned int numpages) +{ + unsigned long addr = (unsigned long)page_address(page); + + if (!can_set_direct_map()) + return 0; + + return set_memory_p(addr, numpages); +} + +/* + * Walk the kernel page table looking for the entry that translates @addr. + * Currently secretmemfd is the only usecase for this. + */ +static bool radix_page_present(unsigned long addr) +{ + pgd_t *pgdp; + p4d_t *p4dp, p4d; + pud_t *pudp, pud; + pmd_t *pmdp, pmd; + pte_t *ptep; + + pgdp = pgd_offset_k(addr); + + p4dp = p4d_offset(pgdp, addr); + p4d = READ_ONCE(*p4dp); + if (p4d_none(p4d)) + return false; + + pudp = pud_offset(p4dp, addr); + pud = READ_ONCE(*pudp); + if (pud_none(pud)) + return false; + if (pud_leaf(pud)) + return pud_present(pud); + + pmdp = pmd_offset(pudp, addr); + pmd = READ_ONCE(*pmdp); + if (pmd_none(pmd)) + return false; + if (pmd_leaf(pmd)) + return pmd_present(pmd); + + ptep = pte_offset_kernel(pmdp, addr); + return pte_hw_valid(ptep_get(ptep)); +} + +bool kernel_page_present(struct page *page) +{ + /* + * Nothing ever removes entries from the linear mapping in this + * configuration, so everything is present. + */ + if (!can_set_direct_map()) + return true; + + return radix_page_present((unsigned long)page_address(page)); +} +#endif /* CONFIG_ARCH_HAS_SET_DIRECT_MAP */ + + #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) #ifdef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC void __kernel_map_pages(struct page *page, int numpages, int enable) -- 2.39.5
