The first-level trap state belongs to the GPUVM because the TBA and TMA are mappings in that GPUVM.
User queues and kernel queues using the same GPUVM use the same first-level trap state. There is no need to keep a separate trap object in the file-private data. Move the trap object into struct amdgpu_vm. Create it when the GPUVM is initialized and free it when the GPUVM is destroyed. Create the trap state after releasing the initial VM root reservation. The trap allocation locks the VM internally, so doing this while the VM root is already reserved could lock the same reservation again. For the same reason, free the trap state before reserving the VM root during VM cleanup. Keep the first-level trap ISA buffer shared at the device level. The ISA is the same for all GPUVMs on the same device, while each GPUVM keeps its own trap mapping and TMA state. Update the user queue path to get the first-level TBA and TMA from its GPUVM. Suggested-by: Christian König <[email protected]> Suggested-by: Alex Deucher <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lancelot Six <[email protected]> Cc: James Zhu <[email protected]> Cc: Lijo Lazar <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Pierre-Eric Pelloux-Prayer <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> Acked-by: Christian König <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 62 ++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 4 ++ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 8 +-- 4 files changed, 61 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 201c167577c6..bc623eeee4bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -1460,7 +1460,7 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) int num_xcc, r; fpriv = (struct amdgpu_fpriv *)filp->driver_priv; - if (!amdgpu_trap_is_enabled(adev) || !fpriv->trap_obj) + if (!amdgpu_trap_is_enabled(adev) || !fpriv->vm.trap_obj) return -EOPNOTSUPP; num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id); cwsr_info.ctl_stack_size = @@ -1604,12 +1604,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) amdgpu_evf_mgr_init(&fpriv->evf_mgr); - r = amdgpu_trap_alloc(adev, - &fpriv->vm, - &fpriv->trap_obj); - if (r == -EOPNOTSUPP) - r = 0; - amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev); file_priv->driver_priv = fpriv; @@ -1679,7 +1673,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, } amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr); - amdgpu_trap_free(adev, &fpriv->vm, &fpriv->trap_obj); if (pasid) amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 2ba33212d507..eafd4367d3b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -45,6 +45,7 @@ #include "amdgpu_xgmi.h" #include "amdgpu_dma_buf.h" #include "amdgpu_res_cursor.h" +#include "amdgpu_trap.h" #include "kfd_svm.h" /** @@ -2644,6 +2645,7 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int32_t xcp_id) { + struct amdgpu_bo_va_mapping *mapping, *tmp; struct amdgpu_bo *root_bo; struct amdgpu_bo_vm *root; int r, i; @@ -2652,6 +2654,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) vm->reserved_vmid[i] = NULL; + vm->trap_obj = NULL; + amdgpu_vm_bo_status_init(&vm->kernel); amdgpu_vm_bo_status_init(&vm->always_valid); spin_lock_init(&vm->individual_lock); @@ -2718,10 +2722,28 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, dev_dbg(adev->dev, "Failed to create task info for VM\n"); amdgpu_bo_unreserve(vm->root.bo); + + if (amdgpu_trap_is_enabled(adev)) { + r = amdgpu_trap_alloc(adev, vm, &vm->trap_obj); + if (r) + goto error_free_trap; + } + amdgpu_bo_unref(&root_bo); return 0; +error_free_trap: + amdgpu_bo_reserve(root_bo, true); + + list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { + list_del(&mapping->list); + amdgpu_vm_free_mapping(adev, vm, mapping, NULL); + } + + amdgpu_vm_put_task_info(vm->task_info); + dma_fence_put(vm->last_update); + error_free_root: amdgpu_vm_pt_free_root(adev, vm); amdgpu_bo_unreserve(vm->root.bo); @@ -2866,22 +2888,39 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); + /* + * Free trap state before reserving root. amdgpu_trap_free() + * uses drm_exec to lock the page directory internally; calling + * it after amdgpu_bo_reserve(root) would deadlock on the same + * ww_mutex. + */ + if (vm->trap_obj) + amdgpu_trap_free(adev, vm, &vm->trap_obj); + root = amdgpu_bo_ref(vm->root.bo); amdgpu_bo_reserve(root, true); + /* Wait for the jobs which might still use the reserved VMID */ if (vm->reserved_vmid[AMDGPU_GFXHUB(0)]) - dma_resv_wait_timeout(root->tbo.base.resv, DMA_RESV_USAGE_READ, - false, MAX_SCHEDULE_TIMEOUT); + dma_resv_wait_timeout(root->tbo.base.resv, + DMA_RESV_USAGE_READ, + false, + MAX_SCHEDULE_TIMEOUT); + dma_fence_wait(vm->last_unlocked, false); dma_fence_put(vm->last_unlocked); + dma_fence_wait(vm->last_tlb_flush, false); + /* Make sure that all fence callbacks have completed */ dma_fence_lock_irqsave(vm->last_tlb_flush, flags); dma_fence_unlock_irqrestore(vm->last_tlb_flush, flags); + dma_fence_put(vm->last_tlb_flush); list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { - if (mapping->flags & AMDGPU_VM_PAGE_PRT && prt_fini_needed) { + if (mapping->flags & AMDGPU_VM_PAGE_PRT && + prt_fini_needed) { amdgpu_vm_prt_fini(adev, vm); prt_fini_needed = false; } @@ -2893,16 +2932,18 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_vm_pt_free_root(adev, vm); amdgpu_bo_unreserve(root); amdgpu_bo_unref(&root); + WARN_ON(vm->root.bo); amdgpu_vm_fini_entities(vm); - if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { + if (!RB_EMPTY_ROOT(&vm->va.rb_root)) dev_err(adev->dev, "still active bo inside vm\n"); - } + rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va.rb_root, rb) { - /* Don't remove the mapping here, we don't want to trigger a + /* + * Don't remove the mapping here, we don't want to trigger a * rebalance and the tree is about to be destroyed anyway. */ list_del(&mapping->list); @@ -2911,9 +2952,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) dma_fence_put(vm->last_update); - for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) { + for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) amdgpu_vmid_free_spm(adev, vm, i); - } ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move); @@ -2922,14 +2962,16 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) dev_warn(adev->dev, "VM memory stats for proc %s(%d) task %s(%d) is non-zero when fini\n", - ti->process_name, ti->task.pid, ti->task.comm, ti->tgid); + ti->process_name, ti->task.pid, + ti->task.comm, ti->tgid); } if (vm->task_info && vm->task_info->task.pid) { int major, minor; amdgpu_vm_render_devt(adev, &major, &minor); - trace_amdgpu_deregister_pid(vm->task_info->task.pid, major, minor); + trace_amdgpu_deregister_pid(vm->task_info->task.pid, + major, minor); } amdgpu_vm_put_task_info(vm->task_info); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index d8e17f493f61..3201397eeec5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -43,6 +43,7 @@ struct amdgpu_bo_va; struct amdgpu_job; struct amdgpu_bo_list_entry; struct amdgpu_bo_vm; +struct amdgpu_trap_obj; /* * GPUVM handling @@ -385,6 +386,9 @@ struct amdgpu_vm { /* cached fault info */ struct amdgpu_vm_fault_info fault_info; + + /* First-level trap state for this GPUVM; NULL if unavailable. */ + struct amdgpu_trap_obj *trap_obj; }; struct amdgpu_vm_manager { diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index 67fccdb64f14..aa532e5a72dc 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -153,10 +153,10 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue) queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo); queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; - if (fpriv->trap_obj) { - queue_input.tba_addr = fpriv->trap_obj->tba_gpu_va_addr; - queue_input.tma_addr = fpriv->trap_obj->tma_gpu_va_addr; - queue_input.trap_en = !amdgpu_trap_has_dbg_wa(adev); + if (fpriv->vm.trap_obj) { + queue_input.tba_addr = fpriv->vm.trap_obj->tba_gpu_va_addr; + queue_input.tma_addr = fpriv->vm.trap_obj->tma_gpu_va_addr; + queue_input.trap_en = !amdgpu_trap_has_dbg_wa(adev); } if (mes->use_rs64mem) { -- 2.34.1
