Introduce IO page table framework support for AMD IOMMU v2 page table. Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpa...@amd.com> --- drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu_types.h | 2 + drivers/iommu/amd/io_pgtable_v2.c | 239 ++++++++++++++++++++++++++++ drivers/iommu/io-pgtable.c | 1 + include/linux/io-pgtable.h | 2 + 5 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 drivers/iommu/amd/io_pgtable_v2.c
diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile index a935f8f4b974..773d8aa00283 100644 --- a/drivers/iommu/amd/Makefile +++ b/drivers/iommu/amd/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_AMD_IOMMU_V2) += iommu_v2.o diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 6937e3674a16..25062eb86c8b 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -265,6 +265,7 @@ * 512GB Pages are not supported due to a hardware bug */ #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) +#define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 12) | (1ULL << 30)) /* Bit value definition for dte irq remapping fields*/ #define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6) @@ -503,6 +504,7 @@ struct amd_io_pgtable { int mode; u64 *root; atomic64_t pt_root; /* pgtable root and pgtable mode */ + struct mm_struct v2_mm; }; /* diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c new file mode 100644 index 000000000000..b0b6ba2d8d35 --- /dev/null +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPU-agnostic AMD IO page table v2 allocator. + * + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * Author: Suravee Suthikulpanit <suravee.suthikulpa...@amd.com> + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include <linux/atomic.h> +#include <linux/bitops.h> +#include <linux/io-pgtable.h> +#include <linux/kernel.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/dma-mapping.h> +#include <linux/mmu_context.h> + +#include <asm/barrier.h> +#include <asm/pgalloc.h> + +#include "amd_iommu_types.h" +#include "amd_iommu.h" + +static pte_t *fetch_pte(struct amd_io_pgtable *pgtable, + unsigned long iova, + unsigned long *page_size) +{ + int level; + pte_t *ptep; + + ptep = lookup_address_in_mm(&pgtable->v2_mm, iova, &level); + if (!ptep || pte_none(*ptep) || (level == PG_LEVEL_NONE)) + return NULL; + + *page_size = PTE_LEVEL_PAGE_SIZE(level-1); + return ptep; +} + +static pte_t *v2_pte_alloc_map(struct mm_struct *mm, unsigned long vaddr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = pgd_offset(mm, vaddr); + p4d = p4d_alloc(mm, pgd, vaddr); + if (!p4d) + return NULL; + pud = pud_alloc(mm, p4d, vaddr); + if (!pud) + return NULL; + pmd = pmd_alloc(mm, pud, vaddr); + if (!pmd) + return NULL; + pte = pte_alloc_map(mm, pmd, vaddr); + return pte; +} + +static int iommu_v2_map_page(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + struct protection_domain *dom = io_pgtable_ops_to_domain(ops); + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); + pte_t *pte; + int ret, i, count; + bool updated = false; + unsigned long o_iova = iova; + unsigned long pte_pgsize; + + BUG_ON(!IS_ALIGNED(iova, size) || !IS_ALIGNED(paddr, size)); + + ret = -EINVAL; + if (!(prot & IOMMU_PROT_MASK)) + goto out; + + count = PAGE_SIZE_PTE_COUNT(size); + + for (i = 0; i < count; ++i, iova += PAGE_SIZE, paddr += PAGE_SIZE) { + pte = fetch_pte(pgtable, iova, &pte_pgsize); + if (!pte || pte_none(*pte)) { + pte = v2_pte_alloc_map(&dom->iop.v2_mm, iova); + if (!pte) + goto out; + } else { + updated = true; + } + set_pte(pte, __pte((paddr & PAGE_MASK)|_PAGE_PRESENT|_PAGE_USER)); + if (prot & IOMMU_PROT_IW) + *pte = pte_mkwrite(*pte); + } + + if (updated) { + if (count > 1) + amd_iommu_flush_tlb(&dom->domain, 0); + else + amd_iommu_flush_page(&dom->domain, 0, o_iova); + } + + ret = 0; +out: + return ret; +} + +static unsigned long iommu_v2_unmap_page(struct io_pgtable_ops *ops, + unsigned long iova, + size_t size, + struct iommu_iotlb_gather *gather) +{ + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); + unsigned long vaddr_end, vaddr_next; + unsigned long long unmapped; + unsigned long pte_pgsize; + pte_t *ptep; + + BUG_ON(!is_power_of_2(size)); + + unmapped = 0; + vaddr_next = iova; + vaddr_end = iova + size; + + for (; iova < vaddr_end; iova = vaddr_next) { + ptep = fetch_pte(pgtable, iova, &pte_pgsize); + if (!ptep || pte_none(*ptep)) + return 0; + pte_unmap(ptep); + unmapped += pte_pgsize; + vaddr_next = (iova & PAGE_MASK) + pte_pgsize; + } + + BUG_ON(unmapped && !is_power_of_2(unmapped)); + return unmapped; +} + +static phys_addr_t iommu_v2_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) +{ + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); + unsigned long offset_mask, pte_pgsize; + u64 __pte; + pte_t *ptep; + + if (pgtable->mode == PAGE_MODE_NONE) + return iova; + + ptep = fetch_pte(pgtable, iova, &pte_pgsize); + + if (!ptep || pte_none(*ptep)) + return 0; + + offset_mask = pte_pgsize - 1; + __pte = __sme_clr(ptep->pte & PM_ADDR_MASK); + + return (__pte & ~offset_mask) | (iova & offset_mask); +} + +/* + * ---------------------------------------------------- + */ +static void v2_tlb_flush_all(void *cookie) +{ +} + +static void v2_tlb_flush_walk(unsigned long iova, size_t size, + size_t granule, void *cookie) +{ +} + +static void v2_tlb_add_page(struct iommu_iotlb_gather *gather, + unsigned long iova, size_t granule, + void *cookie) +{ +} + +static const struct iommu_flush_ops v2_flush_ops = { + .tlb_flush_all = v2_tlb_flush_all, + .tlb_flush_walk = v2_tlb_flush_walk, + .tlb_add_page = v2_tlb_add_page, +}; + +static void v2_free_pgtable(struct io_pgtable *iop) +{ + struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); + struct protection_domain *pdom; + pgd_t *pgd; + struct mm_struct *mm; + + pdom = container_of(pgtable, struct protection_domain, iop); + if (!(pdom->flags & PD_IOMMUV2_MASK)) + return; + + /* Update data structure */ + mm = &pdom->iop.v2_mm; + pgd = mm->pgd; + pgd_free(mm, pgd); + + /* Make changes visible to IOMMUs */ + amd_iommu_domain_update(pdom); + amd_iommu_domain_clear_gcr3(&pdom->domain, 0); +} + +/* + * Assume protection_domain already setup at this point + */ +static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + int ret; + struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); + struct protection_domain *pdom = (struct protection_domain *)cookie; + struct mm_struct *mm = &pdom->iop.v2_mm; + + mm->pgd = pgd_alloc(mm); + if (!mm->pgd) + return NULL; + + ret = amd_iommu_domain_set_gcr3(&pdom->domain, 0, __pa(mm->pgd)); + if (ret) + return NULL; + + pgtable->iop.ops.map = iommu_v2_map_page; + pgtable->iop.ops.unmap = iommu_v2_unmap_page; + pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys; + + cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2, + cfg->ias = IOMMU_IN_ADDR_BIT_SIZE, + cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, + cfg->tlb = &v2_flush_ops; + + return &pgtable->iop; +} + +struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = { + .alloc = v2_alloc_pgtable, + .free = v2_free_pgtable, +}; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 6e9917ce980f..6494657e4a34 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -26,6 +26,7 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { #endif #ifdef CONFIG_AMD_IOMMU [AMD_IOMMU_V1] = &io_pgtable_amd_iommu_v1_init_fns, + [AMD_IOMMU_V2] = &io_pgtable_amd_iommu_v2_init_fns, #endif }; diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index a4c9ca2c31f1..17951204126e 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -16,6 +16,7 @@ enum io_pgtable_fmt { ARM_V7S, ARM_MALI_LPAE, AMD_IOMMU_V1, + AMD_IOMMU_V2, IO_PGTABLE_NUM_FMTS, }; @@ -250,5 +251,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns; #endif /* __IO_PGTABLE_H */ -- 2.17.1 _______________________________________________ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu