MES owns kernel queue VMIDs (1..first_kfd_vmid-1) but does not program SQ_SHADER_TBA/TMA for them. On GFX11+ hardware MES maps kernel queues via ADD_QUEUE with map_legacy_kq=1 but does not set trap handler state. On GFX10 and earlier HWS-based hardware, the driver programs trap registers via SRBM select for KFD queues but no equivalent exists for driver-managed kernel queue VMIDs.
Add a vmhub callback program_kernel_trap_vmids() so each gfxhub version can write SQ_SHADER_TBA/TMA for kernel VMIDs. The TBA points to the device-level CWSR ISA BO. The TMA is set to the fixed per-VM virtual address AMDGPU_VA_RESERVED_TRAP_START — each VM maps its own kq_tma_bo there, so per-VM isolation is handled entirely by page tables without needing to reprogram the register per job or per submission. The per-VM kq_tma_bo is a small GTT BO allocated at VM creation time (parallel to page table allocation) and mapped read-only into the GPU VM at AMDGPU_VA_RESERVED_TRAP_START. The kernel CPU writes the second-level handler address into it via kq_tma_map when userspace calls SET_L2_TRAP. The first-level CWSR handler reads this address to chain to the second-level handler when a shader exception fires. This design is: - Per-VM BO (not device-level) — same model as page tables - Fixed VA in each VM's address space — same VA, different physical BO - Read-only from GPU — kernel CPU updates it via CPU mapping - Treat allocation/free lifecycle identical to page tables Suggested-by: Christian König <[email protected]> Suggested-by: Alexander Deucher <[email protected]> Cc: Lijo Lazar <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> Change-Id: I9ce352157c4aa84099cef926cba61264781e8ad9 --- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 80 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 7 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 9 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 13 ++++ 5 files changed, 110 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h index 3ca187f5ade8..5624a5ab5c62 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h @@ -115,6 +115,7 @@ struct amdgpu_vmhub_funcs { void (*print_l2_protection_fault_status)(struct amdgpu_device *adev, uint32_t status); uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type); + void (*program_kernel_trap_vmids)(struct amdgpu_device *adev); }; struct amdgpu_vmhub { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 623cac6781be..e913488ca3fa 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -263,6 +263,7 @@ int amdgpu_trap_init(struct amdgpu_device *adev) amdgpu_trap_cwsr_init_save_area_info(adev, trap_info); adev->trap_info = no_free_ptr(trap_info); + amdgpu_trap_program_kernel_vmids(adev); return 0; } @@ -277,6 +278,85 @@ void amdgpu_trap_fini(struct amdgpu_device *adev) adev->trap_info = NULL; } +void amdgpu_trap_program_kernel_vmids(struct amdgpu_device *adev) +{ + struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; + + if (!amdgpu_trap_is_enabled(adev)) + return; + if (!hub->vmhub_funcs || !hub->vmhub_funcs->program_kernel_trap_vmids) + return; + + hub->vmhub_funcs->program_kernel_trap_vmids(adev); +} + +int amdgpu_trap_vm_kq_tma_alloc(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + void *cpu_addr; + uint64_t va; + int r; + + dma_resv_assert_held(vm->root.bo->tbo.base.resv); + + r = amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &vm->kq_tma_bo, + NULL, &cpu_addr); + if (r) + return r; + + if (vm->kq_tma_bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK) + iosys_map_set_vaddr_iomem(&vm->kq_tma_map, + (void __iomem *)cpu_addr); + else + iosys_map_set_vaddr(&vm->kq_tma_map, cpu_addr); + + vm->kq_tma_va = amdgpu_vm_bo_add(adev, vm, vm->kq_tma_bo); + if (!vm->kq_tma_va) { + r = -ENOMEM; + goto err_free_bo; + } + + va = AMDGPU_VA_RESERVED_TRAP_START(adev) & AMDGPU_GMC_HOLE_MASK; + r = amdgpu_vm_bo_map(adev, vm->kq_tma_va, va, 0, + AMDGPU_GPU_PAGE_SIZE, + AMDGPU_VM_PAGE_READABLE); + if (r) + goto err_del_va; + + r = amdgpu_vm_bo_update(adev, vm->kq_tma_va, false); + if (r) + goto err_del_va; + + return 0; + +err_del_va: + amdgpu_vm_bo_del(adev, vm->kq_tma_va); + vm->kq_tma_va = NULL; +err_free_bo: + amdgpu_bo_free_kernel(&vm->kq_tma_bo, NULL, NULL); + return r; +} + +void amdgpu_trap_vm_kq_tma_free(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + uint64_t va; + + if (!vm->kq_tma_bo) + return; + + dma_resv_assert_held(vm->root.bo->tbo.base.resv); + + if (vm->kq_tma_va) { + va = AMDGPU_VA_RESERVED_TRAP_START(adev) & AMDGPU_GMC_HOLE_MASK; + amdgpu_vm_bo_unmap(adev, vm->kq_tma_va, va); + amdgpu_vm_bo_del(adev, vm->kq_tma_va); + vm->kq_tma_va = NULL; + } + amdgpu_bo_free_kernel(&vm->kq_tma_bo, NULL, NULL); +} + static int amdgpu_trap_map_region(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj *cwsr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index 7f83174a4742..9be5035abd1c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -155,4 +155,11 @@ int amdgpu_trap_set_trap_debug_flag(struct amdgpu_device *adev, struct amdgpu_trap_obj *cwsr_obj, bool enabled); +/* Kernel queue trap handler — per-VM TMA and register programming */ +void amdgpu_trap_program_kernel_vmids(struct amdgpu_device *adev); +int amdgpu_trap_vm_kq_tma_alloc(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +void amdgpu_trap_vm_kq_tma_free(struct amdgpu_device *adev, + struct amdgpu_vm *vm); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index de3ef9ce2234..f5e228d4bfb5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2640,6 +2640,12 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (r) goto error_free_root; + if (amdgpu_trap_is_enabled(adev)) { + r = amdgpu_trap_vm_kq_tma_alloc(adev, vm); + if (r) + goto error_free_root; + } + r = amdgpu_vm_create_task_info(vm); if (r) dev_dbg(adev->dev, "Failed to create task info for VM\n"); @@ -2774,6 +2780,9 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_vm_free_mapping(adev, vm, mapping, NULL); } + if (vm->kq_tma_bo) + amdgpu_trap_vm_kq_tma_free(adev, vm); + amdgpu_vm_pt_free_root(adev, vm); amdgpu_bo_unreserve(root); amdgpu_bo_unref(&root); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index dd825e179979..064f95a83790 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -25,6 +25,7 @@ #define __AMDGPU_VM_H__ #include <linux/idr.h> +#include <linux/iosys-map.h> #include <linux/kfifo.h> #include <linux/rbtree.h> #include <drm/gpu_scheduler.h> @@ -488,6 +489,18 @@ struct amdgpu_vm { /* cached fault info */ struct amdgpu_vm_fault_info fault_info; + + /* + * Per-VM kernel queue first-level TMA BO. + * Allocated at VM init, freed at VM fini — same lifecycle as page tables. + * Mapped read-only at AMDGPU_VA_RESERVED_TRAP_START in the GPU VM. + * SQ_SHADER_TMA for all kernel VMIDs points to this fixed VA; per-VM + * isolation is via page tables mapping different physical BOs there. + * CPU kernel writes second-level handler address via kq_tma_map. + */ + struct amdgpu_bo *kq_tma_bo; + struct amdgpu_bo_va *kq_tma_va; + struct iosys_map kq_tma_map; }; struct amdgpu_vm_manager { -- 2.34.1
