From: Matthew Brost <[email protected]> In PASID mode the device accesses user mappings of a BO through the IOMMU, but cannot take page faults, so the driver faults the CPU pages in with HMM before submitting a job and waits for the job from the MMU notifier. The HMM PFNs themselves are never used.
The open coded implementation of this has a few problems: - Each mapping embeds a struct hmm_range, and keeps the PFN array it points at allocated for the lifetime of the mapping, although the PFNs are dropped as soon as the fault returns. - The per mapping invalid flag duplicates what the notifier sequence number already tracks, and must be kept in sync with it. - The notifier lock is taken in write mode on the populate path. - Submission locks all reservations, finds the first BO with an invalidated mapping, unlocks everything, faults that single BO in and starts over, once per invalidated BO. drm_gpusvm already implements the fault and retry loop, including faulting without a DMA mapping (&drm_gpusvm_ctx.no_dma_map), and its pages only mode does not require the driver to use drm_gpusvm ranges. Use it: - Replace the device notifier_lock with a pages only struct drm_gpusvm and its notifier lock. - Replace the embedded hmm_range and the invalid/unmapped flags with a struct drm_gpusvm_pages. A mapping is stale when drm_gpusvm_pages_read_retry() says so and it is not unmapped. - Fault in all stale mappings with drm_gpusvm_get_pages() before locking the reservations, then check that none went stale again under the notifier lock, which is held until the job fences are published, as before. - The invalidate callback only bumps the sequence number and marks the mapping unmapped on MMU_NOTIFY_UNMAP under the write lock, before waiting for the jobs. A temporary PFN array is now allocated per fault instead of keeping one per mapping, which also removes the race above. Assisted-by: LLM Signed-off-by: Matthew Brost <[email protected]> [Lizhi: Merge with latest code] Signed-off-by: Lizhi Hou <[email protected]> --- drivers/accel/amdxdna/Kconfig | 2 +- drivers/accel/amdxdna/aie2_ctx.c | 31 +-- drivers/accel/amdxdna/aie4_ctx.c | 29 +-- drivers/accel/amdxdna/amdxdna_gem.c | 312 +++++++++++++----------- drivers/accel/amdxdna/amdxdna_gem.h | 23 +- drivers/accel/amdxdna/amdxdna_pci_drv.c | 15 +- drivers/accel/amdxdna/amdxdna_pci_drv.h | 3 +- 7 files changed, 213 insertions(+), 202 deletions(-) diff --git a/drivers/accel/amdxdna/Kconfig b/drivers/accel/amdxdna/Kconfig index eddff490b4f75..3721eb580b71a 100644 --- a/drivers/accel/amdxdna/Kconfig +++ b/drivers/accel/amdxdna/Kconfig @@ -7,10 +7,10 @@ config DRM_ACCEL_AMDXDNA depends on PCI && HAS_IOMEM depends on X86_64 depends on AMD_PMF || !AMD_PMF + select DRM_GPUSVM select DRM_SCHED select DRM_GEM_SHMEM_HELPER select FW_LOADER - select HMM_MIRROR help Choose this option to enable support for NPU integrated into AMD client CPUs like AMD Ryzen AI 300 Series. AMD NPU can be used to diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index e1c0e0689b12e..552c30ea6e387 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -1116,8 +1116,7 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, struct amdxdna_dev *xdna = hwctx->client->xdna; struct ww_acquire_ctx acquire_ctx; struct dma_fence_chain *chain; - struct amdxdna_gem_obj *abo; - unsigned long timeout = 0; + unsigned long deadline; int ret, i; ret = down_interruptible(&hwctx->priv->job_sem); @@ -1140,7 +1139,15 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, goto free_chain; } + deadline = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); retry: + for (i = 0; i < job->bo_cnt; i++) { + ret = amdxdna_gem_umap_populate(to_xdna_obj(job->bos[i]), + deadline); + if (ret) + goto cleanup_job; + } + ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); if (ret) { XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); @@ -1156,23 +1163,11 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, } } - down_read(&xdna->notifier_lock); + drm_gpusvm_notifier_lock(&xdna->gpusvm); for (i = 0; i < job->bo_cnt; i++) { - abo = to_xdna_obj(job->bos[i]); - if (abo->mem.map_invalid) { - up_read(&xdna->notifier_lock); + if (amdxdna_gem_umap_stale(to_xdna_obj(job->bos[i]))) { + drm_gpusvm_notifier_unlock(&xdna->gpusvm); drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); - if (!timeout) { - timeout = jiffies + - msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - } else if (time_after(jiffies, timeout)) { - ret = -ETIME; - goto cleanup_job; - } - - ret = amdxdna_populate_range(abo); - if (ret) - goto cleanup_job; goto retry; } } @@ -1192,7 +1187,7 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); mutex_unlock(&hwctx->priv->io_lock); - up_read(&xdna->notifier_lock); + drm_gpusvm_notifier_unlock(&xdna->gpusvm); drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); aie2_job_put(job); diff --git a/drivers/accel/amdxdna/aie4_ctx.c b/drivers/accel/amdxdna/aie4_ctx.c index a3f0a69b93b75..0db7dcdf106ba 100644 --- a/drivers/accel/amdxdna/aie4_ctx.c +++ b/drivers/accel/amdxdna/aie4_ctx.c @@ -1059,8 +1059,7 @@ static int fence_job_bos(struct amdxdna_dev *xdna, struct amdxdna_sched_job *job { struct ww_acquire_ctx acquire_ctx; struct drm_gem_object **objs; - struct amdxdna_gem_obj *abo; - unsigned long timeout = 0; + unsigned long deadline; u32 cnt = 0; int ret; u32 i; @@ -1076,7 +1075,13 @@ static int fence_job_bos(struct amdxdna_dev *xdna, struct amdxdna_sched_job *job for (i = 0; i < job->aie4_cmd_bo_cnt; i++) add_lock_obj(objs, &cnt, to_gobj(job->aie4_cmd_bos[i])); + deadline = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); retry: + for (i = 0; i < cnt; i++) { + ret = amdxdna_gem_umap_populate(to_xdna_obj(objs[i]), deadline); + if (ret) + goto free_objs; + } ret = drm_gem_lock_reservations(objs, cnt, &acquire_ctx); if (ret) { XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); @@ -1091,23 +1096,11 @@ static int fence_job_bos(struct amdxdna_dev *xdna, struct amdxdna_sched_job *job } } - down_read(&xdna->notifier_lock); + drm_gpusvm_notifier_lock(&xdna->gpusvm); for (i = 0; i < cnt; i++) { - abo = to_xdna_obj(objs[i]); - if (abo->mem.map_invalid) { - up_read(&xdna->notifier_lock); + if (amdxdna_gem_umap_stale(to_xdna_obj(objs[i]))) { + drm_gpusvm_notifier_unlock(&xdna->gpusvm); drm_gem_unlock_reservations(objs, cnt, &acquire_ctx); - if (!timeout) { - timeout = jiffies + - msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - } else if (time_after(jiffies, timeout)) { - ret = -ETIME; - goto free_objs; - } - - ret = amdxdna_populate_range(abo); - if (ret) - goto free_objs; goto retry; } } @@ -1115,7 +1108,7 @@ static int fence_job_bos(struct amdxdna_dev *xdna, struct amdxdna_sched_job *job job->out_fence = dma_fence_get(job->fence); for (i = 0; i < cnt; i++) dma_resv_add_fence(objs[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); - up_read(&xdna->notifier_lock); + drm_gpusvm_notifier_unlock(&xdna->gpusvm); unlock: drm_gem_unlock_reservations(objs, cnt, &acquire_ctx); diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index 09e48c794e1b0..57d87fa98d671 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -232,37 +232,46 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, { struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier); struct amdxdna_gem_obj *abo = mapp->abo; - struct amdxdna_dev *xdna; + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + bool unreg; if (!mmu_notifier_range_blockable(range)) return false; - if (mapp->unmapped) - return true; - - xdna = to_xdna_dev(to_gobj(abo)->dev); XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d", - mapp->range.start, mapp->range.end, abo->type); + mapp->start, mapp->end, abo->type); - down_write(&xdna->notifier_lock); - abo->mem.map_invalid = true; - mapp->invalid = true; - mmu_interval_set_seq(&mapp->notifier, cur_seq); - up_write(&xdna->notifier_lock); + down_write(&xdna->gpusvm.notifier_lock); + mmu_interval_set_seq(mni, cur_seq); + /* + * Flag the mapping as gone before the wait below, so that a submission + * racing with the unmap does not try to fault it in again. + */ + unreg = !mapp->pages.flags.unmapped && range->event == MMU_NOTIFY_UNMAP; + if (unreg) + drm_gpusvm_pages_set_unmapped(&xdna->gpusvm, &mapp->pages, 1); + up_write(&xdna->gpusvm.notifier_lock); /* * The pages must not be released while the device may still access * them, so this wait must be neither interruptible nor time limited. + * + * Every invalidation of the mapping has to wait, including the ones + * which find it already flagged: the flag is set above before the wait, + * so it does not tell whether the jobs of a concurrent invalidation + * have finished. */ dma_resv_wait_timeout(to_gobj(abo)->resv, DMA_RESV_USAGE_BOOKKEEP, false, MAX_SCHEDULE_TIMEOUT); - if (range->event == MMU_NOTIFY_UNMAP) { - down_write(&xdna->notifier_lock); - mapp->unmapped = true; + /* + * Queue the unregistration only once the jobs are done: it drops the + * last reference of the mapping, and mmu_interval_notifier_remove() + * waits for this callback, which would pin the single worker of the + * ordered WQ_MEM_RECLAIM notifier_wq for as long as a job runs. + */ + if (unreg) queue_work(xdna->notifier_wq, &abo->hmm_unreg_work); - up_write(&xdna->notifier_lock); - } return true; } @@ -271,12 +280,25 @@ static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = { .invalidate = amdxdna_hmm_invalidate, }; -static inline bool compare_range(struct amdxdna_umap *mapp, - struct mm_struct *mm, - unsigned long start, unsigned long end) +static bool amdxdna_umap_match(struct amdxdna_umap *mapp, + struct mm_struct *mm, + unsigned long start, unsigned long end) { - return (!mapp->unmapped && mapp->notifier.mm == mm && - mapp->range.start == start && mapp->range.end == end); + return !mapp->pages.flags.unmapped && mapp->notifier.mm == mm && + mapp->start == start && mapp->end == end; +} + +static bool amdxdna_umap_stale(struct drm_gpusvm *gpusvm, + struct amdxdna_umap *mapp) +{ + /* + * The pages of a BO whose mappings mmap() populated upfront are never + * faulted in. Neither are the ones of a mapping which is gone: device + * accesses through it are rejected by the IOMMU, so there is nothing + * to fault in for it. + */ + return mapp->abo->mem.map_invalid && !mapp->pages.flags.unmapped && + drm_gpusvm_pages_read_retry(gpusvm, &mapp->pages, &mapp->notifier); } static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, @@ -285,15 +307,28 @@ static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); struct amdxdna_umap *mapp; - down_write(&xdna->notifier_lock); + down_write(&xdna->gpusvm.notifier_lock); list_for_each_entry(mapp, &abo->mem.umap_list, node) { - if (!compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) + if (!amdxdna_umap_match(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) continue; - mapp->unmapped = true; + drm_gpusvm_pages_set_unmapped(&xdna->gpusvm, &mapp->pages, 1); queue_work(xdna->notifier_wq, &abo->hmm_unreg_work); } - up_write(&xdna->notifier_lock); + up_write(&xdna->gpusvm.notifier_lock); +} + +static void amdxdna_umap_release(struct kref *ref) +{ + struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); + + mmu_interval_notifier_remove(&mapp->notifier); + kfree(mapp); +} + +static void amdxdna_umap_put(struct amdxdna_umap *mapp) +{ + kref_put(&mapp->refcnt, amdxdna_umap_release); } static void amdxdna_hmm_unreg_umaps(struct amdxdna_gem_obj *abo, bool force) @@ -302,121 +337,119 @@ static void amdxdna_hmm_unreg_umaps(struct amdxdna_gem_obj *abo, bool force) struct amdxdna_umap *mapp, *tmp; LIST_HEAD(dead); - down_write(&xdna->notifier_lock); + down_write(&xdna->gpusvm.notifier_lock); list_for_each_entry_safe(mapp, tmp, &abo->mem.umap_list, node) { - if (!force && !mapp->unmapped) + if (!force && !mapp->pages.flags.unmapped) continue; - mapp->unmapped = true; + drm_gpusvm_pages_set_unmapped(&xdna->gpusvm, &mapp->pages, 1); list_move(&mapp->node, &dead); } if (list_empty(&abo->mem.umap_list)) abo->mem.uva = AMDXDNA_INVALID_ADDR; - up_write(&xdna->notifier_lock); + up_write(&xdna->gpusvm.notifier_lock); list_for_each_entry_safe(mapp, tmp, &dead, node) amdxdna_umap_put(mapp); } -static void amdxdna_umap_release(struct kref *ref) +static void amdxdna_hmm_unreg_work(struct work_struct *work) { - struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); - - mmu_interval_notifier_remove(&mapp->notifier); - mutex_destroy(&mapp->lock); - kvfree(mapp->range.hmm_pfns); - kfree(mapp); -} + struct amdxdna_gem_obj *abo = container_of(work, struct amdxdna_gem_obj, + hmm_unreg_work); -void amdxdna_umap_put(struct amdxdna_umap *mapp) -{ - kref_put(&mapp->refcnt, amdxdna_umap_release); + amdxdna_hmm_unreg_umaps(abo, false); } -int amdxdna_populate_range(struct amdxdna_gem_obj *abo) +/** + * amdxdna_gem_umap_populate() - Fault in the invalidated user mappings of a BO + * @abo: The BO. + * @deadline: Jiffies after which to give up. + * + * The device accesses user mappings through the PASID and cannot take page + * faults, so the CPU pages of every mapping of @abo invalidated since it was + * last faulted are faulted in again here. This is only a snapshot, the caller + * must check amdxdna_gem_umap_stale() under the notifier lock before using + * the mappings and retry if it fails. + * + * Two hardware contexts may fault the same mapping at the same time. That is + * left unserialized: with &drm_gpusvm_ctx.no_dma_map the only state recorded + * for a mapping is its notifier sequence, and the loser of such a race can + * only store an older sequence, which makes amdxdna_gem_umap_stale() report + * stale and the caller fault again. + * + * Return: 0 on success, -ETIME if @deadline passed, negative error code on + * other failures. + */ +int amdxdna_gem_umap_populate(struct amdxdna_gem_obj *abo, + unsigned long deadline) { struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct drm_gpusvm *gpusvm = &xdna->gpusvm; struct amdxdna_umap *mapp; - unsigned long timeout; - struct mm_struct *mm; - bool found; + bool found, unmapped; int ret; - timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); -again: - found = false; - down_write(&xdna->notifier_lock); - list_for_each_entry(mapp, &abo->mem.umap_list, node) { - /* - * Skip entries that have already been unmapped. - * - * If userspace unmaps the address and later submits I/O using - * it, the IOMMU will reject the access and report a fault. - * Ignore such entries here. - */ - if (mapp->unmapped) - continue; + for (;;) { + struct drm_gpusvm_ctx ctx = { .no_dma_map = 1 }; - if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { - found = true; - break; + found = false; + drm_gpusvm_notifier_lock(gpusvm); + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (amdxdna_umap_stale(gpusvm, mapp) && + kref_get_unless_zero(&mapp->refcnt)) { + found = true; + break; + } } - } - - if (!found) { - /* - * This also covers the case where all mappings have been - * removed. There are no invalid mappings left to process. - * Any subsequent I/O using the unmapped address will be - * rejected by the IOMMU. - */ - abo->mem.map_invalid = false; - up_write(&xdna->notifier_lock); - return 0; - } - - up_write(&xdna->notifier_lock); - - mm = mapp->notifier.mm; - if (!mmget_not_zero(mm)) { - amdxdna_umap_put(mapp); - return -EFAULT; - } + drm_gpusvm_notifier_unlock(gpusvm); + if (!found) + return 0; - mutex_lock(&mapp->lock); - ret = hmm_range_fault_unlocked_timeout(&mapp->range, timeout); - if (ret) { - mutex_unlock(&mapp->lock); - goto put_mm; - } + if (time_after(jiffies, deadline)) { + amdxdna_umap_put(mapp); + return -ETIME; + } - down_write(&xdna->notifier_lock); - if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { - up_write(&xdna->notifier_lock); - mutex_unlock(&mapp->lock); + ctx.read_only = mapp->read_only; + ret = drm_gpusvm_get_pages(gpusvm, &mapp->pages, 1, + mapp->notifier.mm, &mapp->notifier, + mapp->start, mapp->end, &ctx); + if (ret == -EFAULT) { + drm_gpusvm_notifier_lock(gpusvm); + unmapped = mapp->pages.flags.unmapped; + drm_gpusvm_notifier_unlock(gpusvm); + if (unmapped) + ret = 0; + } amdxdna_umap_put(mapp); - mmput(mm); - goto again; + if (ret) + return ret == -EBUSY ? -ETIME : ret; } - mapp->invalid = false; - up_write(&xdna->notifier_lock); - mutex_unlock(&mapp->lock); - amdxdna_umap_put(mapp); - mmput(mm); - goto again; - -put_mm: - amdxdna_umap_put(mapp); - mmput(mm); - return ret == -EBUSY ? -ETIME : ret; } -static void amdxdna_hmm_unreg_work(struct work_struct *work) +/** + * amdxdna_gem_umap_stale() - Check whether a BO needs its user mappings faulted + * @abo: The BO. + * + * Must be called with the notifier lock held, which must be kept held until + * the fences of the jobs using @abo are published in its reservation object. + * + * Return: True if amdxdna_gem_umap_populate() must be called for @abo. + */ +bool amdxdna_gem_umap_stale(struct amdxdna_gem_obj *abo) { - struct amdxdna_gem_obj *abo = container_of(work, struct amdxdna_gem_obj, - hmm_unreg_work); + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct amdxdna_umap *mapp; - amdxdna_hmm_unreg_umaps(abo, false); + lockdep_assert_held(&xdna->gpusvm.notifier_lock); + + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (amdxdna_umap_stale(&xdna->gpusvm, mapp)) + return true; + } + + return false; } static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, @@ -426,7 +459,6 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, unsigned long len = vma->vm_end - vma->vm_start; unsigned long addr = vma->vm_start; struct amdxdna_umap *mapp; - unsigned long nr_pages; int ret; /* @@ -441,38 +473,34 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, return 0; } - down_read(&xdna->notifier_lock); + down_read(&xdna->gpusvm.notifier_lock); list_for_each_entry(mapp, &abo->mem.umap_list, node) { - if (compare_range(mapp, current->mm, addr, addr + len)) { - up_read(&xdna->notifier_lock); + if (amdxdna_umap_match(mapp, current->mm, addr, addr + len)) { + up_read(&xdna->gpusvm.notifier_lock); return 0; } } - up_read(&xdna->notifier_lock); + up_read(&xdna->gpusvm.notifier_lock); mapp = kzalloc_obj(*mapp); if (!mapp) return -ENOMEM; - nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT; - mapp->range.hmm_pfns = kvzalloc_objs(*mapp->range.hmm_pfns, nr_pages); - if (!mapp->range.hmm_pfns) { - ret = -ENOMEM; - goto free_map; - } + /* + * Never faulted, so stale until the first amdxdna_gem_umap_populate(), + * should the pages of the BO have to be faulted in at all. + */ + drm_gpusvm_init_pages(&mapp->pages, &xdna->ddev); + mapp->start = addr; + mapp->end = addr + len; - mapp->range.notifier = &mapp->notifier; - mapp->range.start = vma->vm_start; - mapp->range.end = vma->vm_end; /* * Access permissions are fixed at mmap() time. Changing them later - * with mprotect() is not supported: the range keeps requesting the + * with mprotect() is not supported: the mapping keeps requesting the * original permissions, so the application may see a fault failure * or an IOMMU fault. */ - mapp->range.default_flags = HMM_PFN_REQ_FAULT; - if (vma->vm_flags & VM_WRITE) - mapp->range.default_flags |= HMM_PFN_REQ_WRITE; + mapp->read_only = !(vma->vm_flags & VM_WRITE); mapp->abo = abo; kref_init(&mapp->refcnt); @@ -483,23 +511,17 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, &amdxdna_hmm_ops); if (ret) { XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret); - goto free_pfns; + kfree(mapp); + return ret; } - mutex_init(&mapp->lock); - down_write(&xdna->notifier_lock); + down_write(&xdna->gpusvm.notifier_lock); if (list_empty(&abo->mem.umap_list)) abo->mem.uva = addr; list_add_tail(&mapp->node, &abo->mem.umap_list); - up_write(&xdna->notifier_lock); + up_write(&xdna->gpusvm.notifier_lock); return 0; - -free_pfns: - kvfree(mapp->range.hmm_pfns); -free_map: - kfree(mapp); - return ret; } static struct amdxdna_gem_obj * @@ -547,21 +569,13 @@ static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj) amdxdna_gem_destroy_obj(abo); } -static void amdxdna_mark_mapp_invalid(struct amdxdna_gem_obj *abo, - struct vm_area_struct *vma) +static void amdxdna_mark_map_invalid(struct amdxdna_gem_obj *abo) { struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); - struct amdxdna_umap *mapp; - down_write(&xdna->notifier_lock); + down_write(&xdna->gpusvm.notifier_lock); abo->mem.map_invalid = true; - list_for_each_entry(mapp, &abo->mem.umap_list, node) { - if (compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) { - mapp->invalid = true; - break; - } - } - up_write(&xdna->notifier_lock); + up_write(&xdna->gpusvm.notifier_lock); } static int amdxdna_remap_pages(struct amdxdna_gem_obj *abo, @@ -628,7 +642,7 @@ static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo, } if (!(vma->vm_flags & VM_PFNMAP)) - amdxdna_mark_mapp_invalid(abo, vma); + amdxdna_mark_map_invalid(abo); /* Drop the reference drm_gem_mmap_obj() acquired.*/ drm_gem_object_put(to_gobj(abo)); diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h index 5c429869992e8..2deead107d6b1 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.h +++ b/drivers/accel/amdxdna/amdxdna_gem.h @@ -7,19 +7,19 @@ #define _AMDXDNA_GEM_H_ #include <drm/drm_gem_shmem_helper.h> -#include <linux/hmm.h> +#include <drm/drm_gpusvm.h> #include <linux/iova.h> #include "amdxdna_pci_drv.h" struct amdxdna_umap { struct mmu_interval_notifier notifier; - struct hmm_range range; + struct drm_gpusvm_pages pages; + unsigned long start; + unsigned long end; + bool read_only; struct amdxdna_gem_obj *abo; struct list_head node; struct kref refcnt; - bool invalid; - bool unmapped; - struct mutex lock; /* protect range */ }; struct amdxdna_mem { @@ -27,10 +27,16 @@ struct amdxdna_mem { u64 dma_addr; size_t size; struct list_head umap_list; + /* + * True, if the pages of the user mappings have to be faulted in before + * the device accesses them. A mapping which mmap() populated upfront, + * which is a VM_PFNMAP one, must not be faulted in: hmm_range_fault() + * rejects such a VMA. + */ bool map_invalid; /* * Cache the first mmap uva as PASID addr, which can be accessed by driver - * without taking notifier_lock. + * without taking the gpusvm notifier lock. */ u64 uva; }; @@ -103,8 +109,9 @@ static inline u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *abo) abo->mem.dma_addr : amdxdna_gem_uva(abo); } -void amdxdna_umap_put(struct amdxdna_umap *mapp); -int amdxdna_populate_range(struct amdxdna_gem_obj *abo); +int amdxdna_gem_umap_populate(struct amdxdna_gem_obj *abo, + unsigned long deadline); +bool amdxdna_gem_umap_stale(struct amdxdna_gem_obj *abo); void amdxdna_gem_heap_free(struct amdxdna_client *client, struct amdxdna_gem_obj *abo); struct drm_gem_object * diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c index f233eb70a2957..18bd7bdd12969 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.c +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -363,6 +363,7 @@ static void amdxdna_xdna_drm_release(struct drm_device *drm, void *res) struct amdxdna_dev *xdna = res; amdxdna_carveout_fini(xdna); + drm_gpusvm_fini(&xdna->gpusvm); } static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -389,18 +390,18 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) return ret; - init_rwsem(&xdna->notifier_lock); + ret = drm_gpusvm_init(&xdna->gpusvm, "amdxdna", NULL, 0, 0, 0, NULL, + NULL, 0); + if (ret) + return ret; + INIT_LIST_HEAD(&xdna->client_list); pci_set_drvdata(pdev, xdna); ret = drmm_add_action(ddev, amdxdna_xdna_drm_release, xdna); - if (ret) + if (ret) { + drm_gpusvm_fini(&xdna->gpusvm); return ret; - - if (IS_ENABLED(CONFIG_LOCKDEP)) { - fs_reclaim_acquire(GFP_KERNEL); - might_lock(&xdna->notifier_lock); - fs_reclaim_release(GFP_KERNEL); } ret = amdxdna_iommu_init(xdna); diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index 27593190dcf67..7663b883d4f8b 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -7,6 +7,7 @@ #define _AMDXDNA_PCI_DRV_H_ #include <drm/amdxdna_accel.h> +#include <drm/drm_gpusvm.h> #include <drm/drm_mm.h> #include <drm/drm_print.h> #include <linux/iommu.h> @@ -118,7 +119,7 @@ struct amdxdna_dev { struct list_head client_list; struct mutex client_lock; /* client_list */ struct amdxdna_drm_query_firmware_version fw_ver; - struct rw_semaphore notifier_lock; /* for mmu notifier*/ + struct drm_gpusvm gpusvm; /* notifier lock for user mappings */ struct workqueue_struct *notifier_wq; struct iommu_group *group; -- 2.34.1
