On 8/27/26 10:13 AM, Eric Huang wrote:
The KFD/userq software oversubscription-notify timers added to
replace GFX11 MES firmware's oversubscription_timer call
amdgpu_mes_notify_unmap_queue(), which takes mes->mutex_hidden,
the same lock add/remove_hw_queue() use. Under SR-IOV this call
can block up to 9s (vs 2.1s bare metal), so MES round-trip latency
goes up (world-switch contention) it stalls re-mapping queues
and times out KFDQMTest.AllCpQueues.
Keep the firmware timer enabled for SR-IOV VFs and skip the
software timers there; the issue this works around is bare-metal
only.
fixes: "drm/amdkfd: workaround 100% gpu usage issue for gfx11"
Signed-off-by: Eric Huang <[email protected]>
Reviewed-by: Mario Limonciello (AMD) <[email protected]>> ---
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c | 6 ++++++
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 9 +++++++++
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 11 +++++++++--
3 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 5cf1bb1c2085..e5f8a81c3a3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -1208,6 +1208,9 @@ static void
amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
*/
void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
{
+ if (amdgpu_sriov_vf(adev))
+ return;
+
if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
return;
@@ -1224,6 +1227,9 @@ void amdgpu_mes_userq_queue_mapped(struct amdgpu_device
*adev)
*/
void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
{
+ if (amdgpu_sriov_vf(adev))
+ return;
+
if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
return;
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index 33ff1afd7c4c..c81e4b8616d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -1024,6 +1024,15 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes
*mes)
mes_set_hw_res_pkt.use_different_vmid_compute = 1;
mes_set_hw_res_pkt.enable_reg_active_poll = 1;
mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
+ /*
+ * Bare metal: disabled here, KFD arms a software replacement only
+ * while oversubscribed (see kfd_device_queue_manager.c). SR-IOV
+ * guests keep the firmware timer instead of the KFD-side one, since
+ * the notify round-trip is much slower under SR-IOV and would
+ * otherwise stall remapping queues.
+ */
+ if (amdgpu_sriov_vf(adev))
+ mes_set_hw_res_pkt.oversubscription_timer = 50;
if (adev->mes.use_rs64mem)
mes_set_hw_res_pkt.use_rs64mem_for_proc_gang_ctx = 1;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index b37aed34a72c..f37bfeeb4bf4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -282,8 +282,15 @@ static int add_queue_mes(struct device_queue_manager *dqm,
struct queue *q,
return r;
}
- /* GFX11: start notify timer only once when oversubscription begins */
- if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+ /*
+ * GFX11: start notify timer only once when oversubscription begins.
+ * Skip under SR-IOV: MES round-trips are far slower there, and this
+ * notify call shares mes->mutex_hidden with add/remove_hw_queue, so
+ * a slow notify can stall remapping queues. SR-IOV guests keep MES's
+ * own firmware oversubscription timer instead.
+ */
+ if (!amdgpu_sriov_vf(adev) &&
+ KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
dqm->active_cp_queue_count > get_cp_queues_num(dqm))
queue_delayed_work(system_wq, &dqm->notify_unmap_work,
