Implement program_kernel_trap_vmids() in each gfxhub version to write SQ_SHADER_TBA_LO/HI and SQ_SHADER_TMA_LO/HI registers for kernel queue VMIDs (1..first_kfd_vmid-1) using SRBM select.
Only kernel queue VMIDs are programmed here. User queue VMIDs (first_kfd_vmid..15) are handled by MES via the ADD_QUEUE packet's trap_handler_addr field and must not be touched by the driver. The TBA address points to the device-level CWSR ISA BO (isa_bo). The TMA address points to the device-level scratch BO (kq_tma_bo). Both are pinned GTT BOs and cannot be evicted. Addresses are stored as addr >> 8 to match the hardware register format (256-byte aligned). TRAP_EN is set in TBA_HI to enable trap handling for each VMID. WARN_ON is used to catch alignment regressions at development time. The null check on gfx.funcs->select_me_pipe_q guards against calls before GFX IP is fully initialized. GFX10 (gfxhub_v2_0): uses mm-prefixed registers. GFX11 (gfxhub_v3_0, gfxhub_v11_5_0): uses reg-prefixed registers. GFX12 (gfxhub_v12_0): uses reg-prefixed registers. GFX12.1 (gfxhub_v12_1): multi-XCC, iterates over all XCC instances. The function is called at two points: 1. amdgpu_trap_init() — first boot, after ISA and TMA BOs are ready 2. setup_vmid_config() — GPU resume, after GART registers are restored Suggested-by: Christian König <[email protected]> Cc: Alexander Deucher <[email protected]> Cc: Timur Kristof <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 43 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 34 ++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 37 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 34 ++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 34 ++++++++++++++++ 5 files changed, 182 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 652eea6eae4a..32d651be9ad3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -23,6 +23,7 @@ #include "amdgpu.h" #include "gfxhub_v11_5_0.h" +#include "amdgpu_trap.h" #include "gc/gc_11_5_0_offset.h" #include "gc/gc_11_5_0_sh_mask.h" @@ -290,6 +291,44 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +/* + * MES owns kernel VMIDs but does not program trap handler registers. + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level + * CWSR handler is active for kernel queue VMIDs. Required for RADV + * debugging (Valve/Steam Deck) and future Navi ray tracing on kernel queues. + */ +static void gfxhub_v11_5_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = amdgpu_bo_gpu_offset(adev->trap_info->kq_tma_bo); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + /* Program VMIDs 1..first_kfd_vmid-1 (kernel queue range only). + * User queue VMIDs (first_kfd_vmid..15) are programmed by MES. + */ + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -337,6 +376,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v11_5_0_program_kernel_trap_vmids(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) @@ -459,6 +501,7 @@ static void gfxhub_v11_5_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v11_5_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v11_5_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v11_5_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v11_5_0_program_kernel_trap_vmids, }; static void gfxhub_v11_5_0_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 6cbf837d50dd..ffab4a25ec6c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -28,6 +28,7 @@ #include "gc/gc_12_0_0_sh_mask.h" #include "soc24_enum.h" #include "soc15_common.h" +#include "amdgpu_trap.h" #define regGCVM_L2_CNTL3_DEFAULT 0x80120007 #define regGCVM_L2_CNTL4_DEFAULT 0x000000c1 @@ -295,6 +296,35 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void gfxhub_v12_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = amdgpu_bo_gpu_offset(adev->trap_info->kq_tma_bo); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +372,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_0_program_kernel_trap_vmids(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) @@ -464,6 +497,7 @@ static void gfxhub_v12_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v12_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v12_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v12_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v12_0_program_kernel_trap_vmids, }; static void gfxhub_v12_0_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index 4c2fd1e6616e..de162c5066ea 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -406,6 +407,38 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void gfxhub_v12_1_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = amdgpu_bo_gpu_offset(adev->trap_info->kq_tma_bo); + u32 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); + int i, j; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + for_each_inst(j, xcc_mask) { + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, j); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, j); + mutex_unlock(&adev->srbm_mutex); + } +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -468,6 +501,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_1_program_kernel_trap_vmids(adev); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, @@ -751,6 +787,7 @@ static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device * static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v12_1_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v12_1_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v12_1_program_kernel_trap_vmids, }; static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 9ea593e2c719..52a6ee09b036 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -280,6 +281,35 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void gfxhub_v2_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = amdgpu_bo_gpu_offset(adev->trap_info->kq_tma_bo); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -327,6 +357,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v2_0_program_kernel_trap_vmids(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) @@ -428,6 +461,7 @@ static void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v2_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v2_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v2_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v2_0_program_kernel_trap_vmids, }; static void gfxhub_v2_0_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index 9e6a6e13dec0..287e7ea43287 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -287,6 +288,35 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void gfxhub_v3_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = amdgpu_bo_gpu_offset(adev->trap_info->kq_tma_bo); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -334,6 +364,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v3_0_program_kernel_trap_vmids(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) @@ -456,6 +489,7 @@ static void gfxhub_v3_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v3_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v3_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v3_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v3_0_program_kernel_trap_vmids, }; static void gfxhub_v3_0_init(struct amdgpu_device *adev) -- 2.34.1
