Hi Sidong, can you please check the drm usage stats spec at Documentation/gpu/drm-usage-stats.rst ?
I think you need to conform to the key naming part of it, with a single key with a capacity equal to the number of cores. We also need an entry in Documentation/gpu/drm-usage-stats.rst like the other drivers have. Btw, with a perfmon ioctl, we could have Perfetto integration and benefit from other work such as drm-sched tracepoint visualization. Just saying in case it can help in your use cases. Thanks, Tomeu On Sun, Nov 1, 2026 at 4:27 PM Sidong Yang <[email protected]> wrote: > > There is no way to tell how much of the NPU a client uses, or whether its > jobs reach more than one core. Implement the DRM usage stats: account the > time each job occupies a core and report it per client as > drm-engine-npu<n>, along with drm_show_memory_stats() for the BOs. > > Jobs can outlive the file that submitted them, so the counters are > refcounted and each job holds a reference. The time covers the whole > period a job owns a core, so it also includes handling the completion > interrupt and the gaps between the tasks of a multi-task job. > > The array is sized by rdev->max_cores, the number of core slots, not by > the number of cores bound when the file is opened. A core can still be > probing while a client opens the device, and the IRQ handler indexes the > array with core->index, which stays valid when a core is unbound. A slot > with no core bound reads as zero. > > For example, teflon running a model two operations deep 3001 times gives: > > drm-engine-npu0: 22719385 ns > drm-engine-npu1: 23345296 ns > drm-engine-npu2: 0 ns > > The NPU is busy 15.6% of the wall clock time and one core is never used. > > Signed-off-by: Sidong Yang <[email protected]> > --- > v2: Size the per-core counters by rdev->max_cores, the number of core > slots, instead of the number of cores bound when the file is opened. > Sashiko reported the out-of-bounds read and Gyeyoung Baek reproduced > it with KASAN. > > This depends on commit 6eff1d69dae8 ("accel/rocket: search every core > slot when a core is removed"), which added rdev->max_cores. It is in > mainline through drm-misc-fixes but not yet in drm-misc-next. > > Link to v1: > https://lore.kernel.org/all/[email protected]/ > > drivers/accel/rocket/rocket_drv.c | 60 +++++++++++++++++++++++++++++-- > drivers/accel/rocket/rocket_drv.h | 14 ++++++++ > drivers/accel/rocket/rocket_job.c | 7 ++++ > drivers/accel/rocket/rocket_job.h | 4 +++ > 4 files changed, 83 insertions(+), 2 deletions(-) > > diff --git a/drivers/accel/rocket/rocket_drv.c > b/drivers/accel/rocket/rocket_drv.c > index 2bcfe4ab3c68..58a486206565 100644 > --- a/drivers/accel/rocket/rocket_drv.c > +++ b/drivers/accel/rocket/rocket_drv.c > @@ -3,8 +3,10 @@ > > #include <drm/drm_accel.h> > #include <drm/drm_drv.h> > +#include <drm/drm_file.h> > #include <drm/drm_gem.h> > #include <drm/drm_ioctl.h> > +#include <drm/drm_print.h> > #include <drm/rocket_accel.h> > #include <linux/clk.h> > #include <linux/err.h> > @@ -68,6 +70,25 @@ rocket_iommu_domain_put(struct rocket_iommu_domain *domain) > kref_put(&domain->kref, rocket_iommu_domain_destroy); > } > > +static void > +rocket_engine_usage_destroy(struct kref *kref) > +{ > + kfree(container_of(kref, struct rocket_engine_usage, kref)); > +} > + > +struct rocket_engine_usage * > +rocket_engine_usage_get(struct rocket_file_priv *rocket_priv) > +{ > + kref_get(&rocket_priv->engine_usage->kref); > + return rocket_priv->engine_usage; > +} > + > +void > +rocket_engine_usage_put(struct rocket_engine_usage *usage) > +{ > + kref_put(&usage->kref, rocket_engine_usage_destroy); > +} > + > static int > rocket_open(struct drm_device *dev, struct drm_file *file) > { > @@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file > *file) > } > > rocket_priv->rdev = rdev; > + > + rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage, > + elapsed_ns, rdev->max_cores); > + if (!rocket_priv->engine_usage) { > + ret = -ENOMEM; > + goto err_free; > + } > + kref_init(&rocket_priv->engine_usage->kref); > + > rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev); > if (IS_ERR(rocket_priv->domain)) { > ret = PTR_ERR(rocket_priv->domain); > - goto err_free; > + goto err_put_engine_usage; > } > > file->driver_priv = rocket_priv; > @@ -109,6 +139,8 @@ rocket_open(struct drm_device *dev, struct drm_file *file) > mutex_destroy(&rocket_priv->mm_lock); > drm_mm_takedown(&rocket_priv->mm); > rocket_iommu_domain_put(rocket_priv->domain); > +err_put_engine_usage: > + rocket_engine_usage_put(rocket_priv->engine_usage); > err_free: > kfree(rocket_priv); > err_put_mod: > @@ -125,6 +157,7 @@ rocket_postclose(struct drm_device *dev, struct drm_file > *file) > mutex_destroy(&rocket_priv->mm_lock); > drm_mm_takedown(&rocket_priv->mm); > rocket_iommu_domain_put(rocket_priv->domain); > + rocket_engine_usage_put(rocket_priv->engine_usage); > kfree(rocket_priv); > module_put(THIS_MODULE); > } > @@ -139,7 +172,29 @@ static const struct drm_ioctl_desc > rocket_drm_driver_ioctls[] = { > ROCKET_IOCTL(FINI_BO, fini_bo), > }; > > -DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops); > +static void rocket_show_fdinfo(struct drm_printer *p, struct drm_file *file) > +{ > + struct rocket_device *rdev = to_rocket_device(file->minor->dev); > + struct rocket_file_priv *rocket_priv = file->driver_priv; > + unsigned int core; > + > + /* > + * These are estimates: the time is taken around the whole period a > job > + * occupies a core, so it also covers handling the completion > interrupt > + * and, for a job made of several tasks, the gaps between them. > + */ > + for (core = 0; core < rdev->max_cores; core++) > + drm_printf(p, "drm-engine-npu%u:\t%llu ns\n", core, > + rocket_priv->engine_usage->elapsed_ns[core]); > + > + drm_show_memory_stats(p, file); > +} > + > +static const struct file_operations rocket_accel_driver_fops = { > + .owner = THIS_MODULE, > + DRM_ACCEL_FOPS, > + .show_fdinfo = drm_show_fdinfo, > +}; > > /* > * Rocket driver version: > @@ -150,6 +205,7 @@ static const struct drm_driver rocket_drm_driver = { > .open = rocket_open, > .postclose = rocket_postclose, > .gem_create_object = rocket_gem_create_object, > + .show_fdinfo = rocket_show_fdinfo, > .ioctls = rocket_drm_driver_ioctls, > .num_ioctls = ARRAY_SIZE(rocket_drm_driver_ioctls), > .fops = &rocket_accel_driver_fops, > diff --git a/drivers/accel/rocket/rocket_drv.h > b/drivers/accel/rocket/rocket_drv.h > index 2c673bb99ccc..eff5c2ac54d1 100644 > --- a/drivers/accel/rocket/rocket_drv.h > +++ b/drivers/accel/rocket/rocket_drv.h > @@ -16,6 +16,15 @@ struct rocket_iommu_domain { > struct kref kref; > }; > > +/* > + * Time each core has spent running a client's jobs, in nanoseconds, indexed > by > + * core. Jobs hold a reference, as they can outlive the file that submitted > them. > + */ > +struct rocket_engine_usage { > + struct kref kref; > + u64 elapsed_ns[]; > +}; > + > struct rocket_file_priv { > struct rocket_device *rdev; > > @@ -24,9 +33,14 @@ struct rocket_file_priv { > struct mutex mm_lock; > > struct drm_sched_entity sched_entity; > + > + struct rocket_engine_usage *engine_usage; > }; > > struct rocket_iommu_domain *rocket_iommu_domain_get(struct rocket_file_priv > *rocket_priv); > void rocket_iommu_domain_put(struct rocket_iommu_domain *domain); > > +struct rocket_engine_usage *rocket_engine_usage_get(struct rocket_file_priv > *rocket_priv); > +void rocket_engine_usage_put(struct rocket_engine_usage *usage); > + > #endif > diff --git a/drivers/accel/rocket/rocket_job.c > b/drivers/accel/rocket/rocket_job.c > index ce994748dd5d..c0ffbccfb6cc 100644 > --- a/drivers/accel/rocket/rocket_job.c > +++ b/drivers/accel/rocket/rocket_job.c > @@ -13,6 +13,7 @@ > #include <linux/iommu.h> > #include <linux/platform_device.h> > #include <linux/pm_runtime.h> > +#include <linux/sched/clock.h> > > #include "rocket_core.h" > #include "rocket_device.h" > @@ -265,6 +266,7 @@ static void rocket_job_cleanup(struct kref *ref) > unsigned int i; > > rocket_iommu_domain_put(job->domain); > + rocket_engine_usage_put(job->engine_usage); > > dma_fence_put(job->done_fence); > dma_fence_put(job->inference_done_fence); > @@ -351,6 +353,7 @@ static struct dma_fence *rocket_job_run(struct > drm_sched_job *sched_job) > > scoped_guard(mutex, &core->job_lock) { > core->in_flight_job = job; > + job->start_ns = local_clock(); > rocket_job_hw_submit(core, job); > } > > @@ -379,6 +382,9 @@ static void rocket_job_handle_irq(struct rocket_core > *core) > return; > } > > + > core->in_flight_job->engine_usage->elapsed_ns[core->index] += > + local_clock() - core->in_flight_job->start_ns; > + > iommu_detach_group(NULL, iommu_group_get(core->dev)); > dma_fence_signal(core->in_flight_job->done_fence); > pm_runtime_put_autosuspend(core->dev); > @@ -587,6 +593,7 @@ static int rocket_ioctl_submit_job(struct drm_device > *dev, struct drm_file *file > > rjob->rdev = rdev; > rjob->domain = rocket_iommu_domain_get(file_priv); > + rjob->engine_usage = rocket_engine_usage_get(file_priv); > > ret = drm_sched_job_init(&rjob->base, > &file_priv->sched_entity, > diff --git a/drivers/accel/rocket/rocket_job.h > b/drivers/accel/rocket/rocket_job.h > index 4ae00feec3b9..14aa78aebc99 100644 > --- a/drivers/accel/rocket/rocket_job.h > +++ b/drivers/accel/rocket/rocket_job.h > @@ -38,6 +38,10 @@ struct rocket_job { > > struct rocket_iommu_domain *domain; > > + /* When the job was handed to a core, and where to account the time */ > + u64 start_ns; > + struct rocket_engine_usage *engine_usage; > + > struct kref refcount; > }; > > > base-commit: a1e63ab695ad76850ab9787b9eb6aab695dc2965 > prerequisite-patch-id: 519bcdfdde80d902309c8346f749ebc4bb6b29c0 > -- > 2.53.0 >
