MES owns kernel queue VMIDs (1..first_kfd_vmid-1) but does not program
SQ_SHADER_TBA/TMA for them. On GFX11+ hardware MES maps kernel queues
via ADD_QUEUE with map_legacy_kq=1 but does not set trap handler state.
On GFX10 and earlier HWS-based hardware, the driver programs trap
registers via SRBM select for KFD queues but no equivalent exists for
driver-managed kernel queue VMIDs.
Add a vmhub callback program_kernel_trap_vmids() so each gfxhub version
can write SQ_SHADER_TBA/TMA for kernel VMIDs. The TBA points to the
device-level CWSR ISA BO. The TMA is set to the fixed per-VM virtual
address AMDGPU_VA_RESERVED_TRAP_START — each VM maps its own kq_tma_bo
there, so per-VM isolation is handled entirely by page tables without
needing to reprogram the register per job or per submission.
The per-VM kq_tma_bo is a small GTT BO allocated at VM creation time
(parallel to page table allocation) and mapped read-only into the GPU VM
at AMDGPU_VA_RESERVED_TRAP_START. The kernel CPU writes the second-level
handler address into it via kq_tma_map when userspace calls SET_L2_TRAP.
The first-level CWSR handler reads this address to chain to the
second-level handler when a shader exception fires.
This design is:
- Per-VM BO (not device-level) — same model as page tables
- Fixed VA in each VM's address space — same VA, different
physical BO
- Read-only from GPU — kernel CPU updates it via CPU mapping
- Treat allocation/free lifecycle identical to page tables
Suggested-by: Christian König <[email protected]>
Suggested-by: Alexander Deucher <[email protected]>
Cc: Lijo Lazar <[email protected]>
Cc: Timur Kristóf <[email protected]>
Cc: Samuel Pitoiset <[email protected]>
Cc: Natalie Vock <[email protected]>
Signed-off-by: Srinivasan Shanmugam <[email protected]>
Change-Id: I9ce352157c4aa84099cef926cba61264781e8ad9
---
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 80 ++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 7 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 9 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 13 ++++
5 files changed, 110 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 3ca187f5ade8..5624a5ab5c62 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -115,6 +115,7 @@ struct amdgpu_vmhub_funcs {
void (*print_l2_protection_fault_status)(struct amdgpu_device
*adev,
uint32_t status);
uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t
flush_type);
+ void (*program_kernel_trap_vmids)(struct amdgpu_device *adev);
};
struct amdgpu_vmhub {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
index 623cac6781be..e913488ca3fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c
@@ -263,6 +263,7 @@ int amdgpu_trap_init(struct amdgpu_device *adev)
amdgpu_trap_cwsr_init_save_area_info(adev, trap_info);
adev->trap_info = no_free_ptr(trap_info);
+ amdgpu_trap_program_kernel_vmids(adev);
return 0;
}
@@ -277,6 +278,85 @@ void amdgpu_trap_fini(struct amdgpu_device *adev)
adev->trap_info = NULL;
}
+void amdgpu_trap_program_kernel_vmids(struct amdgpu_device *adev)
+{
+ struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+
+ if (!amdgpu_trap_is_enabled(adev))
+ return;
+ if (!hub->vmhub_funcs ||
!hub->vmhub_funcs->program_kernel_trap_vmids)
+ return;
+
+ hub->vmhub_funcs->program_kernel_trap_vmids(adev);
+}
+
+int amdgpu_trap_vm_kq_tma_alloc(struct amdgpu_device *adev,
+ struct amdgpu_vm *vm)
+{
+ void *cpu_addr;
+ uint64_t va;
+ int r;
+
+ dma_resv_assert_held(vm->root.bo->tbo.base.resv);
+
+ r = amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT, &vm->kq_tma_bo,
+ NULL, &cpu_addr);
+ if (r)
+ return r;
+
+ if (vm->kq_tma_bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+ iosys_map_set_vaddr_iomem(&vm->kq_tma_map,
+ (void __iomem *)cpu_addr);
+ else
+ iosys_map_set_vaddr(&vm->kq_tma_map, cpu_addr);
+
+ vm->kq_tma_va = amdgpu_vm_bo_add(adev, vm, vm->kq_tma_bo);
+ if (!vm->kq_tma_va) {
+ r = -ENOMEM;
+ goto err_free_bo;
+ }
+
+ va = AMDGPU_VA_RESERVED_TRAP_START(adev) & AMDGPU_GMC_HOLE_MASK;
