Add a per-partition scheduler that queues access-window GPU requests and grants GPU access to queued windows in turn.
An uncontended access window can retain access after its timeslice expires. The scheduler requests a yield when the timeslice has expired and another window is waiting and force-closes the active window if it does not yield in time. Use hrtimers for GPU timeslices and stop/close completion deadlines. Start stop and close timers after successful command submission. Retry stop and close commands from workqueue context when submission returns -EAGAIN. Use scheduler reset work to recover from other command errors and completion timeouts. Signed-off-by: Karunika Choo <[email protected]> --- v2: - Use sched_id instead of looking it up via the aw_id for sched to AW operations. - Fix stale timer callback from running on a newer lease generation. - Fix following issues raised by sashiko: - Simplify scheduler tear-down path. - Fix timeout value truncation. - Introduce retry path for commands that fail due to possibility of partition commands being rejected due to transient state transition. - Add check for allocation failure from devm_kasprintf - Rollback failed suspend / resume. - Link: https://lore.kernel.org/all/[email protected]/ v1: - Move grant, stop and close operations from workqueues into IRQ-safe paths. Retains work for scheduler recovery. - Add closing phase and timer to ensure CLOSE_WINDOW completes in due time, otherwise enter recovery handling. - Rename "grant_timeout_us" to "request_timeout" and "yield_timeout_us" to "yield_timeout". Both units are now in milliseconds. - Add ABI documentation .../testing/sysfs-driver-panthor-arbitration | 18 + MAINTAINERS | 1 + drivers/gpu/drm/panthor/arbitration/Makefile | 1 + .../panthor/arbitration/panthor_arbitration.h | 9 + .../arbitration/panthor_arbitration_drv.c | 31 + .../arbitration/panthor_arbitration_sched.c | 838 ++++++++++++++++++ .../arbitration/panthor_arbitration_sched.h | 38 + 7 files changed, 936 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-driver-panthor-arbitration create mode 100644 drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.c create mode 100644 drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.h diff --git a/Documentation/ABI/testing/sysfs-driver-panthor-arbitration b/Documentation/ABI/testing/sysfs-driver-panthor-arbitration new file mode 100644 index 0000000000000..fbd21270a2412 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-panthor-arbitration @@ -0,0 +1,18 @@ +What: /sys/module/panthor_arbitration/parameters/request_timeout +Date: September 2026 +KernelVersion: 7.2.0 +Contact: Arm Ltd. +Description: + Controls how long, in milliseconds, the arbitration scheduler + allows an access window to keep the GPU before asking it to + yield when another access window is waiting. + +What: /sys/module/panthor_arbitration/parameters/yield_timeout +Date: September 2026 +KernelVersion: 7.2.0 +Contact: Arm Ltd. +Description: + Controls how long, in milliseconds, the arbitration scheduler + waits for an access window to yield the GPU after a stop request. + If the timeout expires, the scheduler force-closes the access + window. diff --git a/MAINTAINERS b/MAINTAINERS index bdf3cc1044c3e..4be924fdf6e29 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2400,6 +2400,7 @@ M: Liviu Dudau <[email protected]> L: [email protected] S: Supported T: git https://gitlab.freedesktop.org/drm/misc/kernel.git +F: Documentation/ABI/testing/sysfs-driver-panthor-arbitration F: Documentation/devicetree/bindings/gpu/arm,mali-gen5-am*.yaml F: Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml F: drivers/gpu/drm/ci/xfails/panthor* diff --git a/drivers/gpu/drm/panthor/arbitration/Makefile b/drivers/gpu/drm/panthor/arbitration/Makefile index 2d2684321d978..22ed68948fee3 100644 --- a/drivers/gpu/drm/panthor/arbitration/Makefile +++ b/drivers/gpu/drm/panthor/arbitration/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_DRM_PANTHOR_ARBITRATION) += panthor_arbitration.o panthor_arbitration-y := \ panthor_arbitration_drv.o \ + panthor_arbitration_sched.o \ panthor_partition_control.o \ panthor_resource_group.o diff --git a/drivers/gpu/drm/panthor/arbitration/panthor_arbitration.h b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration.h index 9612bfc6c4417..8d4c0aab12666 100644 --- a/drivers/gpu/drm/panthor/arbitration/panthor_arbitration.h +++ b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration.h @@ -6,8 +6,10 @@ #include <linux/ioport.h> #include <linux/fwnode.h> +#include <linux/types.h> struct device; +struct panthor_arbitration_sched; struct panthor_partition_control; struct panthor_resource_group; @@ -27,6 +29,13 @@ struct panthor_arbitration { /** @rg: Pointer array to resource group data */ struct panthor_resource_group *rg[AM_ARB_MAX_RG_COUNT]; + + /** @sched: Pointer to scheduler data. 1 sched per partition */ + struct panthor_arbitration_sched *sched[AM_ARB_MAX_PC_COUNT]; }; +int panthor_arbitration_on_grant(struct panthor_arbitration *adev, u8 sched_id, u8 aw_id); +int panthor_arbitration_on_stop(struct panthor_arbitration *adev, u8 sched_id); +int panthor_arbitration_on_close(struct panthor_arbitration *adev, u8 sched_id); + #endif diff --git a/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_drv.c b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_drv.c index 0e8ffd3c1e6da..f26dcdcbc9198 100644 --- a/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_drv.c +++ b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_drv.c @@ -11,6 +11,7 @@ #include <linux/of.h> #include "panthor_arbitration.h" +#include "panthor_arbitration_sched.h" #include "panthor_partition_control.h" #include "panthor_resource_group.h" @@ -21,6 +22,10 @@ static int panthor_arbitration_runtime_suspend(struct device *dev) struct panthor_arbitration *adev = dev_get_drvdata(dev); int ret = 0; + ret = panthor_arbitration_sched_suspend(adev); + if (ret) + return ret; + ret = panthor_resource_group_suspend(adev); if (ret) return ret; @@ -45,6 +50,26 @@ static int panthor_arbitration_runtime_resume(struct device *dev) if (ret) return ret; + ret = panthor_arbitration_sched_resume(adev); + if (ret) + return ret; + + return 0; +} + +/* Arbiter to AW events */ +int panthor_arbitration_on_grant(struct panthor_arbitration *adev, u8 sched_id, u8 aw_id) +{ + return 0; +} + +int panthor_arbitration_on_stop(struct panthor_arbitration *adev, u8 sched_id) +{ + return 0; +} + +int panthor_arbitration_on_close(struct panthor_arbitration *adev, u8 sched_id) +{ return 0; } @@ -71,6 +96,10 @@ static int panthor_arbitration_probe(struct platform_device *pdev) if (ret) goto err_disable_pm; + ret = panthor_arbitration_sched_init(adev); + if (ret) + goto err_term; + ret = panthor_partition_control_init(adev); if (ret) goto err_term; @@ -94,6 +123,7 @@ static int panthor_arbitration_probe(struct platform_device *pdev) of_platform_depopulate(dev); err_term: + panthor_arbitration_sched_term(adev); panthor_resource_group_term(adev); panthor_partition_control_term(adev); pm_runtime_put_sync_suspend(dev); @@ -119,6 +149,7 @@ static void panthor_arbitration_remove(struct platform_device *pdev) pm_runtime_disable(adev->dev); + panthor_arbitration_sched_term(adev); panthor_partition_control_term(adev); panthor_resource_group_term(adev); diff --git a/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.c b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.c new file mode 100644 index 0000000000000..23768739e9ef5 --- /dev/null +++ b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.c @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT +/* Copyright 2026 ARM Limited. All rights reserved. */ + +#include <linux/bits.h> +#include <linux/device.h> +#include <linux/kfifo.h> +#include <linux/moduleparam.h> +#include <linux/pm_runtime.h> +#include <linux/time.h> +#include <linux/types.h> + +#include "panthor_arbitration.h" +#include "panthor_arbitration_sched.h" +#include "panthor_resource_group.h" + +static u32 request_timeout = 1 * MSEC_PER_SEC; +static u32 yield_timeout = 1 * MSEC_PER_SEC; + +module_param(request_timeout, uint, 0644); +MODULE_PARM_DESC(request_timeout, "Time the GPU will be granted to the AW in milliseconds."); + +module_param(yield_timeout, uint, 0644); +MODULE_PARM_DESC(yield_timeout, "Time, in milliseconds, within which the AW must yield the GPU."); + +enum arbitration_sched_phase { + ARB_SCHED_PHASE_IDLE = 0, + ARB_SCHED_PHASE_GRANTING, + ARB_SCHED_PHASE_GRANTED, + ARB_SCHED_PHASE_STOPPING, + ARB_SCHED_PHASE_CLOSING, + ARB_SCHED_PHASE_LOST, +}; + +struct panthor_arbitration_timer { + /** @timer: High-resolution timer */ + struct hrtimer timer; + + /** @epoch: The lease generation for which the timer was triggered */ + u32 epoch; +}; + +struct panthor_arbitration_work { + /** @work: Work struct */ + struct work_struct work; + + /** @aw_id: Window for which the work was queued */ + u8 aw_id; + + /** @epoch: The lease generation for which the work was queued */ + u32 epoch; +}; + +enum panthor_arbitration_retry_work { + PANTHOR_ARB_WORK_STOP = 0, + PANTHOR_ARB_WORK_CLOSE, + PANTHOR_ARB_WORK_COUNT, +}; + +struct panthor_arbitration_sched { + /** @dev: Device pointer */ + struct device *dev; + + /** @id: Scheduler index. Sched N maps to partition control N */ + u8 id; + + /** @name: Scheduler name */ + char *name; + + /** @waitqueue: Event wait queue. Not IRQ safe */ + wait_queue_head_t waitqueue; + + /** @lock: Protects scheduler state */ + spinlock_t lock; + + /** @queue: Pending request FIFO */ + DECLARE_KFIFO(queue, u8, 16); + + /** @queued_mask: Mask of currently queued AWs */ + DECLARE_BITMAP(queued_mask, AM_ARB_MAX_AW_COUNT); + + /** @active_aw_id: Currently active AW. Can be -1 */ + s8 active_aw_id; + + /** @phase: Current scheduling phase */ + enum arbitration_sched_phase phase; + + /** @epoch: Current active lease generation */ + u32 epoch; + + /** @wq: Scheduler workqueue to dispatch events */ + struct workqueue_struct *wq; + + /** @reset_work: Work to reset the scheduler */ + struct panthor_arbitration_work reset_work; + + /** @retry_work: Work to retry an operation */ + struct panthor_arbitration_work retry_work; + + /** @pending_retry_work: Retry work that is pending */ + DECLARE_BITMAP(pending_retry_work, PANTHOR_ARB_WORK_COUNT); + + /** @grant_timer: Timer to indicate grant window has lapsed */ + struct panthor_arbitration_timer grant_timer; + + /** @stop_timer: Timer to indicate yield window has lapsed */ + struct panthor_arbitration_timer stop_timer; + + /** @close_timer: Timer to indicate close window has lapsed */ + struct panthor_arbitration_timer close_timer; + + /** @disabled: Scheduler disabled. No further scheduling can happen */ + u32 disabled; + + /** + * @grant_previously_expired: Grant period previously expired but was + * not asked to stop + */ + bool grant_previously_expired; +}; + +static int arb_sched_on_stopped_locked(struct panthor_arbitration_sched *sched, u8 aw_id); + +static inline struct panthor_arbitration * +to_adev(struct panthor_arbitration_sched *sched) +{ + return dev_get_drvdata(sched->dev); +} + +static inline struct panthor_arbitration_work *to_work(struct work_struct *w) +{ + return container_of(w, struct panthor_arbitration_work, work); +} + +static u32 panthor_arbitration_stop_timeout(struct panthor_arbitration_sched *sched) +{ + u64 timeout = (u64)READ_ONCE(request_timeout) + READ_ONCE(yield_timeout); + + if (timeout > (u64)INT_MAX) { + dev_warn_once(sched->dev, "request/yield timeout too large."); + return INT_MAX; + } + + return timeout; +} + +static bool arb_sched_phase_reached(struct panthor_arbitration_sched *sched, + enum arbitration_sched_phase phase) +{ + guard(spinlock_irqsave)(&sched->lock); + + return sched->phase == phase; +} + +/* This method is not IRQ safe as it sleeps. */ +static int arb_sched_wait_phase(struct panthor_arbitration_sched *sched, + enum arbitration_sched_phase phase, + u32 timeout_ms) +{ + if (!wait_event_timeout( + sched->waitqueue, arb_sched_phase_reached(sched, phase), + msecs_to_jiffies(timeout_ms))) { + + if (!arb_sched_phase_reached(sched, phase)) + return -ETIMEDOUT; + } + + return 0; +} + +static void arb_sched_queue_work(struct panthor_arbitration_sched *sched, + struct panthor_arbitration_work *work) +{ + lockdep_assert_held(&sched->lock); + + work->aw_id = sched->active_aw_id; + work->epoch = sched->epoch; + queue_work(sched->wq, &work->work); +} + +static void arb_sched_queue_retry_work(struct panthor_arbitration_sched *sched, + enum panthor_arbitration_retry_work work) +{ + lockdep_assert_held(&sched->lock); + + if (!test_and_set_bit(work, sched->pending_retry_work)) + arb_sched_queue_work(sched, &sched->retry_work); +} + +static int __arb_sched_stop_locked(struct panthor_arbitration_sched *sched) +{ + int ret; + + lockdep_assert_held(&sched->lock); + + ret = panthor_arbitration_on_stop(to_adev(sched), sched->id); + if (ret == -EAGAIN) { + arb_sched_queue_retry_work(sched, PANTHOR_ARB_WORK_STOP); + } else if (ret) { + dev_err(sched->dev, "%s: Failed to send yield to AW%u. %d", + sched->name, sched->active_aw_id, ret); + + arb_sched_queue_work(sched, &sched->reset_work); + } + + return ret; +} + +static void arb_sched_stop_locked(struct panthor_arbitration_sched *sched) +{ + int ret; + + lockdep_assert_held(&sched->lock); + + ret = __arb_sched_stop_locked(sched); + if (!ret) { + if (sched->phase != ARB_SCHED_PHASE_STOPPING) { + dev_warn(sched->dev, "%s: Stop ACKED when not yielding", + sched->name); + return; + } + + sched->stop_timer.epoch = sched->epoch; + hrtimer_start(&sched->stop_timer.timer, + ms_to_ktime(yield_timeout), HRTIMER_MODE_REL); + } +} + +static void arb_sched_grant_locked(struct panthor_arbitration_sched *sched) +{ + int ret; + + lockdep_assert_held(&sched->lock); + + ret = panthor_arbitration_on_grant(to_adev(sched), sched->id, sched->active_aw_id); + if (!ret) { + if (sched->phase != ARB_SCHED_PHASE_GRANTING) { + dev_warn(sched->dev, "%s: Grant ACKED when not granting", + sched->name); + return; + } + + if (sched->disabled) { + sched->phase = ARB_SCHED_PHASE_STOPPING; + arb_sched_stop_locked(sched); + } else { + sched->phase = ARB_SCHED_PHASE_GRANTED; + + sched->grant_timer.epoch = sched->epoch; + hrtimer_start(&sched->grant_timer.timer, + ms_to_ktime(request_timeout), + HRTIMER_MODE_REL); + } + + wake_up_all(&sched->waitqueue); + } else if (ret < 0) { + dev_err(sched->dev, "%s: Failed to grant access to AW%u. %d", + sched->name, sched->active_aw_id, ret); + + arb_sched_queue_work(sched, &sched->reset_work); + } +} + +static int __arb_sched_close_locked(struct panthor_arbitration_sched *sched) +{ + int ret; + + lockdep_assert_held(&sched->lock); + + ret = panthor_arbitration_on_close(to_adev(sched), sched->id); + if (ret == -EAGAIN) { + arb_sched_queue_retry_work(sched, PANTHOR_ARB_WORK_CLOSE); + } else if (ret < 0) { + dev_err(sched->dev, "%s: Failed to close AW%u. %d", sched->name, + sched->active_aw_id, ret); + + arb_sched_queue_work(sched, &sched->reset_work); + } + + return ret; +} + +static void arb_sched_close_locked(struct panthor_arbitration_sched *sched) +{ + int ret; + + lockdep_assert_held(&sched->lock); + + ret = __arb_sched_close_locked(sched); + if (!ret) { + sched->close_timer.epoch = sched->epoch; + hrtimer_start(&sched->close_timer.timer, + ms_to_ktime(yield_timeout), HRTIMER_MODE_REL); + } +} + +static void arb_sched_request_stop_locked(struct panthor_arbitration_sched *sched) +{ + lockdep_assert_held(&sched->lock); + + if (sched->phase != ARB_SCHED_PHASE_GRANTED) + return; + + if (sched->active_aw_id < 0) + return; + + /* + * In the event of a single requesting AW, let it continue to + * lease the GPU until another request comes in. + */ + if (!sched->disabled && kfifo_is_empty(&sched->queue)) { + sched->grant_previously_expired = true; + return; + } + + sched->phase = ARB_SCHED_PHASE_STOPPING; + arb_sched_stop_locked(sched); + + wake_up_all(&sched->waitqueue); +} + +static void arb_sched_request_stop(struct panthor_arbitration_sched *sched) +{ + guard(spinlock_irqsave)(&sched->lock); + + arb_sched_request_stop_locked(sched); +} + +static void arb_sched_force_close_locked(struct panthor_arbitration_sched *sched) +{ + lockdep_assert_held(&sched->lock); + + if (sched->phase == ARB_SCHED_PHASE_IDLE) + return; + + if (sched->active_aw_id < 0) + return; + + sched->phase = ARB_SCHED_PHASE_LOST; + + arb_sched_close_locked(sched); + + wake_up_all(&sched->waitqueue); +} + +static int arb_sched_force_close(struct panthor_arbitration_sched *sched) +{ + scoped_guard(spinlock_irqsave, &sched->lock) + arb_sched_force_close_locked(sched); + + return arb_sched_wait_phase(sched, ARB_SCHED_PHASE_IDLE, + panthor_arbitration_stop_timeout(sched)); +} + +static int arb_sched_next_locked(struct panthor_arbitration_sched *sched) +{ + u8 aw_id; + + lockdep_assert_held(&sched->lock); + + if (sched->disabled) + return 0; + + /* already have a running aw */ + if (sched->active_aw_id >= 0) + return 0; + + /* nothing to schedule */ + if (kfifo_is_empty(&sched->queue)) + return 0; + + if (!kfifo_get(&sched->queue, &aw_id)) { + dev_warn(sched->dev, "%s: queue unexpectedly empty", + sched->name); + return -EINVAL; + } + + clear_bit(aw_id, sched->queued_mask); + + sched->active_aw_id = aw_id; + sched->epoch++; + sched->phase = ARB_SCHED_PHASE_GRANTING; + + arb_sched_grant_locked(sched); + + wake_up_all(&sched->waitqueue); + + return 0; +} + +static int arb_sched_next(struct panthor_arbitration_sched *sched) +{ + guard(spinlock_irqsave)(&sched->lock); + + return arb_sched_next_locked(sched); +} + +static void arb_sched_reset_work(struct work_struct *work) +{ + struct panthor_arbitration_work *w = to_work(work); + struct panthor_arbitration_sched *sched = + container_of(w, struct panthor_arbitration_sched, reset_work); + + guard(spinlock_irqsave)(&sched->lock); + if (sched->epoch != w->epoch) + return; + + if (sched->active_aw_id != w->aw_id) + return; + + dev_warn(sched->dev, "%s: scheduler reset. active_aw=%u", + sched->name, sched->active_aw_id); + + arb_sched_force_close_locked(sched); +} + +static void arb_sched_retry_work(struct work_struct *work) +{ + struct panthor_arbitration_work *w = to_work(work); + struct panthor_arbitration_sched *sched = + container_of(w, struct panthor_arbitration_sched, retry_work); + DECLARE_BITMAP(work_tmp, PANTHOR_ARB_WORK_COUNT); + + guard(spinlock_irqsave)(&sched->lock); + if (bitmap_empty(sched->pending_retry_work, PANTHOR_ARB_WORK_COUNT)) + return; + + bitmap_copy(work_tmp, sched->pending_retry_work, PANTHOR_ARB_WORK_COUNT); + bitmap_zero(sched->pending_retry_work, PANTHOR_ARB_WORK_COUNT); + + if (sched->epoch != w->epoch || sched->active_aw_id != w->aw_id) + return; + + switch (sched->phase) { + case ARB_SCHED_PHASE_STOPPING: + if (test_bit(PANTHOR_ARB_WORK_STOP, work_tmp)) + arb_sched_stop_locked(sched); + break; + case ARB_SCHED_PHASE_CLOSING: + case ARB_SCHED_PHASE_LOST: + if (test_bit(PANTHOR_ARB_WORK_CLOSE, work_tmp)) + arb_sched_close_locked(sched); + break; + default: + break; + } +} + +static enum hrtimer_restart +panthor_arbitration_sched_on_stop_timeout(struct hrtimer *timer) +{ + struct panthor_arbitration_timer *atmr = container_of( + timer, struct panthor_arbitration_timer, timer); + struct panthor_arbitration_sched *sched = container_of( + atmr, struct panthor_arbitration_sched, stop_timer); + + guard(spinlock_irqsave)(&sched->lock); + + if (hrtimer_is_queued(timer)) + return HRTIMER_NORESTART; + + if (sched->epoch != atmr->epoch || + sched->phase != ARB_SCHED_PHASE_STOPPING) + return HRTIMER_NORESTART; + + dev_warn(sched->dev, + "%s: AW%d failed to stop in time. Force closing...", + sched->name, sched->active_aw_id); + arb_sched_force_close_locked(sched); + + return HRTIMER_NORESTART; +} + +static enum hrtimer_restart +panthor_arbitration_sched_on_grant_timeout(struct hrtimer *timer) +{ + struct panthor_arbitration_timer *atmr = container_of( + timer, struct panthor_arbitration_timer, timer); + struct panthor_arbitration_sched *sched = container_of( + atmr, struct panthor_arbitration_sched, grant_timer); + + guard(spinlock_irqsave)(&sched->lock); + + if (hrtimer_is_queued(timer)) + return HRTIMER_NORESTART; + + if (sched->epoch != atmr->epoch) + return HRTIMER_NORESTART; + + arb_sched_request_stop_locked(sched); + + return HRTIMER_NORESTART; +} + +static enum hrtimer_restart +panthor_arbitration_sched_on_close_timeout(struct hrtimer *timer) +{ + struct panthor_arbitration_timer *atmr = container_of( + timer, struct panthor_arbitration_timer, timer); + struct panthor_arbitration_sched *sched = container_of( + atmr, struct panthor_arbitration_sched, close_timer); + + guard(spinlock_irqsave)(&sched->lock); + + if (hrtimer_is_queued(timer)) + return HRTIMER_NORESTART; + + if (sched->epoch != atmr->epoch) + return HRTIMER_NORESTART; + + if (sched->phase != ARB_SCHED_PHASE_CLOSING && + sched->phase != ARB_SCHED_PHASE_LOST) + return HRTIMER_NORESTART; + + dev_warn(sched->dev, + "%s: AW%d failed to close in time. Resetting scheduler...", + sched->name, sched->active_aw_id); + + arb_sched_queue_work(sched, &sched->reset_work); + + return HRTIMER_NORESTART; +} + +int panthor_arbitration_sched_on_request(struct panthor_arbitration_sched *sched, u8 aw_id) +{ + guard(spinlock_irqsave)(&sched->lock); + + if (aw_id >= AM_ARB_MAX_AW_COUNT) + return -EINVAL; + + if (test_bit(aw_id, sched->queued_mask) || + (sched->active_aw_id == aw_id && + sched->phase != ARB_SCHED_PHASE_STOPPING)) { + dev_warn(sched->dev, "%s: AW%u already in request queue.", + sched->name, aw_id); + return -EEXIST; + } + + /* Always put onto queue */ + if (!kfifo_put(&sched->queue, aw_id)) { + dev_err(sched->dev, "%s: request queue unexpectedly full", + sched->name); + return -ENOSPC; + } + set_bit(aw_id, sched->queued_mask); + + /* No active AW, schedule next AW */ + if (sched->active_aw_id < 0) + return arb_sched_next_locked(sched); + + /* + * Grant timer for the current AW has previously expired, request it to + * yield the GPU. + */ + if (sched->grant_previously_expired) + arb_sched_request_stop_locked(sched); + + return 0; +} + +int panthor_arbitration_sched_on_idle(struct panthor_arbitration_sched *sched, u8 aw_id) +{ + guard(spinlock_irqsave)(&sched->lock); + + if (aw_id >= AM_ARB_MAX_AW_COUNT) + return -EINVAL; + + if (sched->active_aw_id != aw_id) + return 0; + + if (sched->phase != ARB_SCHED_PHASE_GRANTED && + sched->phase != ARB_SCHED_PHASE_STOPPING) + return 0; + + sched->phase = ARB_SCHED_PHASE_CLOSING; + arb_sched_close_locked(sched); + + return 0; +} + +static int arb_sched_on_stopped_locked(struct panthor_arbitration_sched *sched, u8 aw_id) +{ + if (aw_id >= AM_ARB_MAX_AW_COUNT) + return -EINVAL; + + if (sched->phase == ARB_SCHED_PHASE_IDLE) + return 0; + + if (sched->active_aw_id != aw_id) + return 0; + + /* retire old lease */ + sched->epoch++; + + sched->grant_previously_expired = false; + sched->active_aw_id = -1; + sched->phase = ARB_SCHED_PHASE_IDLE; + bitmap_zero(sched->pending_retry_work, PANTHOR_ARB_WORK_COUNT); + + wake_up_all(&sched->waitqueue); + + return arb_sched_next_locked(sched); +} + +int panthor_arbitration_sched_on_stopped(struct panthor_arbitration_sched *sched, u8 aw_id) +{ + guard(spinlock_irqsave)(&sched->lock); + + return arb_sched_on_stopped_locked(sched, aw_id); +} + +static void arbitration_sched_disable_work(struct panthor_arbitration_sched *sched) +{ + disable_work_sync(&sched->reset_work.work); + disable_work_sync(&sched->retry_work.work); +} + +static void arbitration_sched_cancel(struct panthor_arbitration_sched *sched) +{ + panthor_arbitration_sched_disable_set(sched, ARB_SCHED_DISABLE_PM); + arbitration_sched_disable_work(sched); + hrtimer_cancel(&sched->grant_timer.timer); + hrtimer_cancel(&sched->stop_timer.timer); + hrtimer_cancel(&sched->close_timer.timer); +} + +static void arbitration_sched_term(struct panthor_arbitration_sched *sched) +{ + panthor_arbitration_sched_disable_set(sched, ARB_SCHED_DISABLE_TERM); + + if (!pm_runtime_status_suspended(sched->dev)) + WARN_ON(panthor_arbitration_sched_stop(sched)); + + arbitration_sched_cancel(sched); +} + +static int arbitration_sched_init(struct panthor_arbitration *adev, int i) +{ + struct device *dev = adev->dev; + struct panthor_arbitration_sched *sched; + + sched = devm_kzalloc(dev, sizeof(*sched), GFP_KERNEL); + if (!sched) + return -ENOMEM; + + sched->name = devm_kasprintf(dev, GFP_KERNEL, "sched%d", i); + if (!sched->name) + return -ENOMEM; + + sched->id = i; + sched->dev = dev; + + sched->wq = devm_alloc_ordered_workqueue(dev, "%s-wq", 0, sched->name); + if (!sched->wq) + return -ENOMEM; + + INIT_WORK(&sched->reset_work.work, arb_sched_reset_work); + INIT_WORK(&sched->retry_work.work, arb_sched_retry_work); + + init_waitqueue_head(&sched->waitqueue); + + spin_lock_init(&sched->lock); + INIT_KFIFO(sched->queue); + sched->active_aw_id = -1; + sched->phase = ARB_SCHED_PHASE_IDLE; + + hrtimer_setup(&sched->grant_timer.timer, + panthor_arbitration_sched_on_grant_timeout, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_setup(&sched->stop_timer.timer, + panthor_arbitration_sched_on_stop_timeout, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_setup(&sched->close_timer.timer, + panthor_arbitration_sched_on_close_timeout, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + adev->sched[i] = sched; + + return 0; +} + +void panthor_arbitration_sched_term(struct panthor_arbitration *adev) +{ + for (int i = 0; i < AM_ARB_MAX_PC_COUNT; i++) { + struct panthor_arbitration_sched *sched = adev->sched[i]; + + if (!sched) + continue; + + arbitration_sched_term(sched); + } +} + +int panthor_arbitration_sched_init(struct panthor_arbitration *adev) +{ + + for (int i = 0; i < AM_ARB_MAX_PC_COUNT; i++) { + int ret = arbitration_sched_init(adev, i); + + if (ret == -ENODEV) + continue; + + if (ret) + return ret; + } + + return 0; +} + +int panthor_arbitration_sched_suspend(struct panthor_arbitration *adev) +{ + int ret; + int i; + + for (i = 0; i < AM_ARB_MAX_PC_COUNT; i++) { + struct panthor_arbitration_sched *sched = adev->sched[i]; + + if (!sched) + continue; + + panthor_arbitration_sched_disable_set(sched, ARB_SCHED_DISABLE_PM); + ret = panthor_arbitration_sched_stop(sched); + if (ret) + goto err_rollback; + } + + return 0; + +err_rollback: + for (; i >= 0; i--) { + struct panthor_arbitration_sched *sched = adev->sched[i]; + + if (!sched) + continue; + + panthor_arbitration_sched_disable_clear(sched, ARB_SCHED_DISABLE_PM); + panthor_arbitration_sched_start(sched); + } + + return ret; +} + +int panthor_arbitration_sched_resume(struct panthor_arbitration *adev) +{ + int ret; + int i; + + for (i = 0; i < AM_ARB_MAX_PC_COUNT; i++) { + struct panthor_arbitration_sched *sched = adev->sched[i]; + + if (!sched) + continue; + + panthor_arbitration_sched_disable_clear(sched, ARB_SCHED_DISABLE_PM); + ret = panthor_arbitration_sched_start(sched); + if (ret) + goto err_rollback; + } + + return 0; + +err_rollback: + for (; i >= 0; i--) { + struct panthor_arbitration_sched *sched = adev->sched[i]; + + if (!sched) + continue; + + panthor_arbitration_sched_disable_set(sched, ARB_SCHED_DISABLE_PM); + panthor_arbitration_sched_stop(sched); + } + + return ret; +} + +/** + * panthor_arbitration_sched_stop - Stop active aw and pause scheduler + * + * @sched: Pointer to arbitration scheduler + * + * This method is not IRQ safe as it waits for on_stopped() to be called. + */ +int panthor_arbitration_sched_stop(struct panthor_arbitration_sched *sched) +{ + int ret; + + arb_sched_request_stop(sched); + + /* + * Provisionally waiting for up to 2x yield_timeout, but path may + * include time taken for panthor_arbitration_on_grant() to complete + * if sched_request_stop() is called while in the GRANTING state. + * + * Realistically, if granting takes more than yield_timeout, HW is in + * bad state either way. Best continue and stop the scheduler. + */ + ret = arb_sched_wait_phase(sched, ARB_SCHED_PHASE_IDLE, + panthor_arbitration_stop_timeout(sched)); + if (ret) { + /* + * TODO: handle the timeout here by escalating the error when + * reset escalation is supported. + */ + ret = arb_sched_force_close(sched); + } + + /* Should no longer have any further activity. Cancel timer. */ + hrtimer_cancel(&sched->grant_timer.timer); + hrtimer_cancel(&sched->stop_timer.timer); + hrtimer_cancel(&sched->close_timer.timer); + flush_workqueue(sched->wq); + + scoped_guard(spinlock_irqsave, &sched->lock) + if (sched->phase == ARB_SCHED_PHASE_IDLE) + return 0; + + return ret; +} + +int panthor_arbitration_sched_start(struct panthor_arbitration_sched *sched) +{ + return arb_sched_next(sched); +} + +void panthor_arbitration_sched_disable_set(struct panthor_arbitration_sched *sched, + enum panthor_arb_sched_disable_reason reason) +{ + guard(spinlock_irqsave)(&sched->lock); + + sched->disabled |= BIT(reason); +} + +void panthor_arbitration_sched_disable_clear(struct panthor_arbitration_sched *sched, + enum panthor_arb_sched_disable_reason reason) +{ + guard(spinlock_irqsave)(&sched->lock); + + sched->disabled &= ~BIT(reason); +} diff --git a/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.h b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.h new file mode 100644 index 0000000000000..4d0f72838d218 --- /dev/null +++ b/drivers/gpu/drm/panthor/arbitration/panthor_arbitration_sched.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 or MIT */ +/* Copyright 2026 ARM Limited. All rights reserved. */ + +#ifndef __PANTHOR_ARBITRATION_SCHED_H__ +#define __PANTHOR_ARBITRATION_SCHED_H__ + +#include <linux/types.h> + +struct panthor_arbitration; +struct panthor_arbitration_sched; + +enum panthor_arb_sched_disable_reason { + /** @ARB_SCHED_DISABLE_PM: scheduler disabled due to PM suspend */ + ARB_SCHED_DISABLE_PM = 0, + + /** @ARB_SCHED_DISABLE_TERM: scheduler disabled for termination */ + ARB_SCHED_DISABLE_TERM, +}; + +int panthor_arbitration_sched_init(struct panthor_arbitration *adev); +void panthor_arbitration_sched_term(struct panthor_arbitration *adev); + +int panthor_arbitration_sched_suspend(struct panthor_arbitration *adev); +int panthor_arbitration_sched_resume(struct panthor_arbitration *adev); + +int panthor_arbitration_sched_stop(struct panthor_arbitration_sched *sched); +int panthor_arbitration_sched_start(struct panthor_arbitration_sched *sched); + +void panthor_arbitration_sched_disable_set(struct panthor_arbitration_sched *sched, + enum panthor_arb_sched_disable_reason reason); +void panthor_arbitration_sched_disable_clear(struct panthor_arbitration_sched *sched, + enum panthor_arb_sched_disable_reason reason); + +int panthor_arbitration_sched_on_request(struct panthor_arbitration_sched *sched, u8 aw_id); +int panthor_arbitration_sched_on_idle(struct panthor_arbitration_sched *sched, u8 aw_id); +int panthor_arbitration_sched_on_stopped(struct panthor_arbitration_sched *sched, u8 aw_id); + +#endif -- 2.43.0
