Add a per-drm_file wait-event manager for render-node events. The manager stores pending event records, supports blocking waits, copies the first matching record to userspace, and enforces single-consumer semantics by removing the record when a waiter consumes it.
Embed the manager in amdgpu_fpriv and tie its lifetime to drm_file. Cc: Alex Deucher <[email protected]> Cc: Christian König <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/Makefile | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 2 + .../gpu/drm/amd/amdgpu/amdgpu_wait_event.c | 288 ++++++++++++++++++ .../gpu/drm/amd/amdgpu/amdgpu_wait_event.h | 85 ++++++ 5 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index feea63b332fa..c883e1840180 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -72,7 +72,8 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o \ amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o amdgpu_dev_coredump.o \ amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \ - amdgpu_eventfd.o + amdgpu_eventfd.o \ + amdgpu_wait_event.o amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 30df0354f44c..950bd617829f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -104,6 +104,7 @@ #include "amdgpu_fdinfo.h" #include "amdgpu_mca.h" #include "amdgpu_aca.h" +#include "amdgpu_wait_event.h" #include "amdgpu_eventfd.h" #include "amdgpu_ras.h" #include "amdgpu_cper.h" @@ -458,13 +459,15 @@ struct amdgpu_fpriv { uint32_t xcp_id; struct amdgpu_eventfd_mgr eventfd_mgr; + struct amdgpu_wait_event_mgr wait_event_mgr; }; struct drm_device; struct drm_file; int amdgpu_eventfd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); - +int amdgpu_wait_event_drm_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv); /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 5e2abf32420f..e99059a4663e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -1509,6 +1509,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) } amdgpu_eventfd_mgr_init(&fpriv->eventfd_mgr); + amdgpu_wait_event_mgr_init(&fpriv->wait_event_mgr); pasid = amdgpu_pasid_alloc(16); if (pasid < 0) { @@ -1608,6 +1609,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, /* Drop all subscriptions before fpriv goes away. */ amdgpu_eventfd_mgr_fini(&fpriv->eventfd_mgr); + amdgpu_wait_event_mgr_fini(&fpriv->wait_event_mgr); pm_runtime_get_sync(dev->dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c new file mode 100644 index 000000000000..c055b201c428 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c @@ -0,0 +1,288 @@ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/jiffies.h> +#include <linux/sched/signal.h> +#include <linux/slab.h> +#include <linux/uaccess.h> + +#include "amdgpu.h" +#include "amdgpu_wait_event.h" + +static long amdgpu_wait_event_to_jiffies(__s64 timeout_ns) +{ + unsigned long long t; + long timeout; + + if (timeout_ns < 0) + return MAX_SCHEDULE_TIMEOUT; + if (timeout_ns == 0) + return 0; + + t = nsecs_to_jiffies(timeout_ns); + if (t > MAX_SCHEDULE_TIMEOUT) + timeout = MAX_SCHEDULE_TIMEOUT - 1; + else + timeout = t; + + return timeout ?: 1; +} + +static bool amdgpu_wait_event_match(const struct drm_amdgpu_wait_event *args, + const struct amdgpu_wait_event_record *rec) +{ + if (rec->data.event_type != args->event_type) + return false; + + if (amdgpu_wait_event_type_is_queue_scoped(args->event_type)) + return rec->data.queue_id == args->queue_id; + + return args->queue_id == 0; +} + +static bool amdgpu_wait_event_has_match(struct amdgpu_wait_event_mgr *mgr, + const struct drm_amdgpu_wait_event *args) +{ + struct amdgpu_wait_event_record *rec; + bool found = false; + unsigned long flags; + + spin_lock_irqsave(&mgr->lock, flags); + list_for_each_entry(rec, &mgr->pending, node) { + if (amdgpu_wait_event_match(args, rec)) { + found = true; + break; + } + } + spin_unlock_irqrestore(&mgr->lock, flags); + + return found; +} + +static struct amdgpu_wait_event_record * +amdgpu_wait_event_pop_match(struct amdgpu_wait_event_mgr *mgr, + const struct drm_amdgpu_wait_event *args) +{ + struct amdgpu_wait_event_record *rec, *tmp, *found = NULL; + unsigned long flags; + + spin_lock_irqsave(&mgr->lock, flags); + list_for_each_entry_safe(rec, tmp, &mgr->pending, node) { + if (amdgpu_wait_event_match(args, rec)) { + list_del(&rec->node); + found = rec; + break; + } + } + spin_unlock_irqrestore(&mgr->lock, flags); + + return found; +} + +static int amdgpu_wait_event_push_common(struct amdgpu_wait_event_mgr *mgr, + struct drm_amdgpu_wait_event_data *data) +{ + struct amdgpu_wait_event_record *rec; + unsigned long flags; + + rec = kzalloc(sizeof(*rec), GFP_ATOMIC); + if (!rec) + return -ENOMEM; + + rec->data = *data; + rec->data.seqno = atomic64_inc_return(&mgr->seqno); + + spin_lock_irqsave(&mgr->lock, flags); + list_add_tail(&rec->node, &mgr->pending); + spin_unlock_irqrestore(&mgr->lock, flags); + + wake_up_interruptible_all(&mgr->wq); + return 0; +} + +void amdgpu_wait_event_mgr_init(struct amdgpu_wait_event_mgr *mgr) +{ + spin_lock_init(&mgr->lock); + init_waitqueue_head(&mgr->wq); + INIT_LIST_HEAD(&mgr->pending); + atomic64_set(&mgr->seqno, 0); + mgr->dead = false; +} + +void amdgpu_wait_event_mgr_fini(struct amdgpu_wait_event_mgr *mgr) +{ + struct amdgpu_wait_event_record *rec, *tmp; + unsigned long flags; + + spin_lock_irqsave(&mgr->lock, flags); + mgr->dead = true; + list_for_each_entry_safe(rec, tmp, &mgr->pending, node) { + list_del(&rec->node); + kfree(rec); + } + spin_unlock_irqrestore(&mgr->lock, flags); + + wake_up_interruptible_all(&mgr->wq); +} + +void amdgpu_wait_event_push_userq_eop(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 status, + u64 data0, u64 data1) +{ + struct drm_amdgpu_wait_event_data data = {}; + + data.event_type = DRM_AMDGPU_EVENT_TYPE_USERQ_EOP; + data.queue_id = queue_id; + data.u.queue.queue_id = queue_id; + data.u.queue.status = status; + data.u.queue.data0 = data0; + data.u.queue.data1 = data1; + + amdgpu_wait_event_push_common(mgr, &data); +} + +void amdgpu_wait_event_push_queue_reset(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 reset_cause, + u64 data0, u64 data1) +{ + struct drm_amdgpu_wait_event_data data = {}; + + data.event_type = DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET; + data.queue_id = queue_id; + data.u.reset.queue_id = queue_id; + data.u.reset.reset_cause = reset_cause; + data.u.reset.data0 = data0; + data.u.reset.data1 = data1; + + amdgpu_wait_event_push_common(mgr, &data); +} + +void amdgpu_wait_event_push_memory_exception(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 fault_status, + u64 va, u64 data0) +{ + struct drm_amdgpu_wait_event_data data = {}; + + data.event_type = DRM_AMDGPU_EVENT_TYPE_MEMORY_EXCEPTION; + data.queue_id = queue_id; + data.u.memory.queue_id = queue_id; + data.u.memory.fault_status = fault_status; + data.u.memory.va = va; + data.u.memory.data0 = data0; + + amdgpu_wait_event_push_common(mgr, &data); +} + +void amdgpu_wait_event_push_scratch(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 error_code, + u64 requested_bytes, + u64 available_bytes) +{ + struct drm_amdgpu_wait_event_data data = {}; + + data.event_type = DRM_AMDGPU_EVENT_TYPE_SCRATCH; + data.queue_id = queue_id; + data.u.scratch.queue_id = queue_id; + data.u.scratch.error_code = error_code; + data.u.scratch.requested_bytes = requested_bytes; + data.u.scratch.available_bytes = available_bytes; + + amdgpu_wait_event_push_common(mgr, &data); +} + +void amdgpu_wait_event_push_gpu_reset(struct amdgpu_wait_event_mgr *mgr, + u32 reset_cause) +{ + struct drm_amdgpu_wait_event_data data = {}; + + data.event_type = DRM_AMDGPU_EVENT_TYPE_GPU_RESET; + data.queue_id = 0; + + data.u.reset.queue_id = 0; + data.u.reset.reset_cause = reset_cause; + data.u.reset.data0 = 0; + data.u.reset.data1 = 0; + + amdgpu_wait_event_push_common(mgr, &data); +} + +int amdgpu_wait_event_drm_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct drm_amdgpu_wait_event *args = data; + struct amdgpu_wait_event_mgr *mgr; + struct amdgpu_wait_event_record *rec; + long timeout; + int ret = 0; + + if (!fpriv) + return -EINVAL; + + if (args->flags || !args->event_type || !args->out_ptr) + return -EINVAL; + + if (args->out_size < sizeof(struct drm_amdgpu_wait_event_data)) + return -EINVAL; + + if (amdgpu_wait_event_type_is_queue_scoped(args->event_type)) { + if (!args->queue_id) + return -EINVAL; + } else { + if (args->queue_id) + return -EINVAL; + } + + mgr = &fpriv->wait_event_mgr; + timeout = amdgpu_wait_event_to_jiffies(args->timeout_ns); + + for (;;) { + rec = amdgpu_wait_event_pop_match(mgr, args); + if (rec) + break; + + if (READ_ONCE(mgr->dead)) + return -EIO; + + if (signal_pending(current)) + return -ERESTARTSYS; + + if (!timeout) + return -ETIME; + + timeout = wait_event_interruptible_timeout( + mgr->wq, + READ_ONCE(mgr->dead) || + amdgpu_wait_event_has_match(mgr, args), + timeout); + if (timeout < 0) + return timeout; + } + + if (copy_to_user(u64_to_user_ptr(args->out_ptr), &rec->data, + sizeof(rec->data))) + ret = -EFAULT; + + kfree(rec); + return ret; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h new file mode 100644 index 000000000000..67de685647c7 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h @@ -0,0 +1,85 @@ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_WAIT_EVENT_H__ +#define __AMDGPU_WAIT_EVENT_H__ + +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/wait.h> +#include <linux/atomic.h> +#include <drm/drm_device.h> +#include <drm/drm_file.h> +#include <uapi/drm/amdgpu_drm.h> + +struct amdgpu_wait_event_record { + struct list_head node; + struct drm_amdgpu_wait_event_data data; +}; + +struct amdgpu_wait_event_mgr { + spinlock_t lock; + wait_queue_head_t wq; + struct list_head pending; + atomic64_t seqno; + bool dead; +}; + +void amdgpu_wait_event_mgr_init(struct amdgpu_wait_event_mgr *mgr); +void amdgpu_wait_event_mgr_fini(struct amdgpu_wait_event_mgr *mgr); + +int amdgpu_wait_event_drm_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +void amdgpu_wait_event_push_userq_eop(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 status, + u64 data0, u64 data1); +void amdgpu_wait_event_push_queue_reset(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 reset_cause, + u64 data0, u64 data1); +void amdgpu_wait_event_push_memory_exception(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 fault_status, + u64 va, u64 data0); +void amdgpu_wait_event_push_scratch(struct amdgpu_wait_event_mgr *mgr, + u32 queue_id, u32 error_code, + u64 requested_bytes, + u64 available_bytes); +void amdgpu_wait_event_push_gpu_reset(struct amdgpu_wait_event_mgr *mgr, + u32 reset_cause); + +static inline bool amdgpu_wait_event_type_is_queue_scoped(u32 event_type) +{ + switch (event_type) { + case DRM_AMDGPU_EVENT_TYPE_USERQ_EOP: + case DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET: + case DRM_AMDGPU_EVENT_TYPE_SCRATCH: + return true; + case DRM_AMDGPU_EVENT_TYPE_MEMORY_EXCEPTION: + case DRM_AMDGPU_EVENT_TYPE_GPU_RESET: + return false; + default: + return false; + } +} + +#endif /* __AMDGPU_WAIT_EVENT_H__ */ \ No newline at end of file -- 2.34.1
