Implement the program_kernel_trap_vmids() vmhub callback for all supported hardware generations. Each implementation programs SQ_SHADER_TBA/TMA for kernel queue VMIDs (1..first_kfd_vmid-1) via SRBM select.
TBA points to the device-level CWSR ISA BO — same binary for all VMIDs. TMA is set to AMDGPU_VA_RESERVED_TRAP_START, the fixed virtual address where each VM's kq_tma_bo is mapped. The hardware resolves this VA through the active VMID's page tables, giving each process its own physical TMA buffer without requiring per-submission register updates. User queue VMIDs (first_kfd_vmid..15) must not be touched here — they are programmed by MES via the ADD_QUEUE packet's trap_handler_addr field. The function is called at two points: 1. amdgpu_trap_init() — first boot, after ISA BO is ready 2. setup_vmid_config() — GPU resume, after GART is restored GFX versions covered: GFX10 (gfxhub_v2_0): mm-prefixed registers, single XCC GFX11 (gfxhub_v3_0): reg-prefixed registers, single XCC GFX11.5 (gfxhub_v11_5_0): reg-prefixed registers, single XCC GFX12 (gfxhub_v12_0): reg-prefixed registers, single XCC GFX12.1 (gfxhub_v12_1): reg-prefixed registers, multi-XCC Suggested-by: Christian König <[email protected]> Suggested-by: Alexander Deucher <[email protected]> Cc: Lijo Lazar <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> Change-Id: I2fa05a8ac96e5fe4b6e5f79dbdf62f32b8dbf96f --- drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 36 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 33 ++++++++++++++++++ 5 files changed, 183 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 652eea6eae4a..b6f43c8274d4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v11_5_0.h" #include "gc/gc_11_5_0_offset.h" @@ -290,6 +291,39 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +/* + * MES owns kernel VMIDs but does not program trap handler registers. + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level + * CWSR handler is active for kernel queue VMIDs. + */ +static void gfxhub_v11_5_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -337,6 +371,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v11_5_0_program_kernel_trap_vmids(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) @@ -459,6 +496,7 @@ static void gfxhub_v11_5_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v11_5_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v11_5_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v11_5_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v11_5_0_program_kernel_trap_vmids, }; static void gfxhub_v11_5_0_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 6cbf837d50dd..5b2f0f59b895 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v12_0.h" #include "gc/gc_12_0_0_offset.h" @@ -295,6 +296,39 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +/* + * MES owns kernel VMIDs but does not program trap handler registers. + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level + * CWSR handler is active for kernel queue VMIDs. + */ +static void gfxhub_v12_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +376,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_0_program_kernel_trap_vmids(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) @@ -464,6 +501,7 @@ static void gfxhub_v12_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v12_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v12_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v12_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v12_0_program_kernel_trap_vmids, }; static void gfxhub_v12_0_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index 4c2fd1e6616e..528917913950 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -406,6 +407,37 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void gfxhub_v12_1_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + u32 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); + int i, j; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + + for_each_inst(j, xcc_mask) { + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, j); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, j); + mutex_unlock(&adev->srbm_mutex); + } +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -468,6 +500,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_1_program_kernel_trap_vmids(adev); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, @@ -751,6 +786,7 @@ static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device * static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v12_1_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v12_1_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v12_1_program_kernel_trap_vmids, }; static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 9ea593e2c719..47a25df2a0c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -280,6 +281,39 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +/* + * MES owns kernel VMIDs but does not program trap handler registers. + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level + * CWSR handler is active for kernel queue VMIDs. + */ +static void gfxhub_v2_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -327,6 +361,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v2_0_program_kernel_trap_vmids(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) @@ -428,6 +465,7 @@ static void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v2_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v2_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v2_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v2_0_program_kernel_trap_vmids, }; static void gfxhub_v2_0_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index 9e6a6e13dec0..2f59de6e9407 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -287,6 +288,34 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void gfxhub_v3_0_program_kernel_trap_vmids(struct amdgpu_device *adev) +{ + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo); + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + int i; + + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -334,6 +363,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v3_0_program_kernel_trap_vmids(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) @@ -456,6 +488,7 @@ static void gfxhub_v3_0_set_fault_enable_default(struct amdgpu_device *adev, static const struct amdgpu_vmhub_funcs gfxhub_v3_0_vmhub_funcs = { .print_l2_protection_fault_status = gfxhub_v3_0_print_l2_protection_fault_status, .get_invalidate_req = gfxhub_v3_0_get_invalidate_req, + .program_kernel_trap_vmids = gfxhub_v3_0_program_kernel_trap_vmids, }; static void gfxhub_v3_0_init(struct amdgpu_device *adev) -- 2.34.1
