From: Benjamin Welton <[email protected]>
kfd_ioctl_profiler takes a similar approach to that of
kfd_ioctl_dbg_trap (which contains debugger related IOCTL
services) where kfd_ioctl_profiler will contain all profiler
related IOCTL services. The IOCTL is designed to be expanded
as needed to support additional profiler functionality.
The current functionality of the IOCTL is to allow for profilers
which need PMC counters from GPU devices to both signal to other
profilers that may be on the system that the device has active PMC
profiling taking place on it (multiple PMC profilers on the same
device can result in corrupted counter data) and to setup the device
to allow for the collection of SQ PMC data on all queues on the device.
For PMC data for the SQ block (such as SQ_WAVES) to be available
to a profiler, mmPERFCOUNT_ENABLE must be set on the queues. When
profiling a single process, the profiler can inject PM4 packets into
each queue to turn on PERFCOUNT_ENABLE. When profiling system wide,
the profiler does not have this option and must have a way to turn
on profiling for queues in which it cannot inject packets into directly.
Accomplishing this requires a few steps:
1. Checking if the user has the necessary permissions to profile system
wide on the device. This check uses the same check that linux perf
uses to determine if a user has the necessary permissions to profile
at this scope (primarily if the process has CAP_SYS_PERFMON or is root).
2. Locking the device for profiling. This is done by setting a lock bit
on the device struct and storing the process that locked the device.
3. Iterating all queues on the device and issuing an MQD Update to enable
perfcounting on the queues.
4. Actions to cleanup if the process exits or releases the lock.
The IOCTL also contains a link to the existing PC Sampling IOCTL as well.
This is per a suggestion that we should potentially remove the PC Sampling
IOCTL to have it be a part of the profiler IOCTL. This is a future change.
In addition, we do expect to expand the profiler IOCTL to include
additional profiler functionality in the future (which necessitates the
use of a version number).
Signed-off-by: Benjamin Welton <[email protected]>
Signed-off-by: Perry Yuan <[email protected]>
Acked-by: Kent Russell <[email protected]>
Reviewed-by: Yifan Zhang <[email protected]>
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 82 +++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_device.c | 4 +
.../drm/amd/amdkfd/kfd_device_queue_manager.c | 25 ++++++
.../drm/amd/amdkfd/kfd_device_queue_manager.h | 2 +
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c | 16 +++-
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c | 14 +++-
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c | 8 +-
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 15 +++-
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c | 11 +++
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 7 ++
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 11 +++
include/uapi/linux/kfd_ioctl.h | 30 +++++++
12 files changed, 217 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 732ad1224a61..dbb111a33678 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -21,6 +21,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
+#include <linux/capability.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/fs.h>
@@ -3204,6 +3205,84 @@ static int kfd_ioctl_create_process(struct file *filep,
struct kfd_process *p, v
return 0;
}
+static inline uint32_t profile_lock_device(struct kfd_process *p,
+ uint32_t gpu_id, uint32_t op)
+{
+ struct kfd_process_device *pdd;
+ struct kfd_dev *kfd;
+ int status = -EINVAL;
+
+ if (!p)
+ return -EINVAL;
+
+ mutex_lock(&p->mutex);
+ pdd = kfd_process_device_data_by_id(p, gpu_id);
+ mutex_unlock(&p->mutex);
+
+ if (!pdd || !pdd->dev || !pdd->dev->kfd)
+ return -EINVAL;
+
+ kfd = pdd->dev->kfd;
+
+ mutex_lock(&kfd->profiler_lock);
+ if (op == 1) {
+ if (!kfd->profiler_process) {
+ kfd->profiler_process = p;
+ status = 0;
+ } else if (kfd->profiler_process == p) {
+ status = -EALREADY;
+ } else {
+ status = -EBUSY;
+ }
+ } else if (op == 0 && kfd->profiler_process == p) {
+ kfd->profiler_process = NULL;
+ status = 0;
+ }
+ mutex_unlock(&kfd->profiler_lock);
+
+ return status;
+}
+
+static inline int kfd_profiler_pmc(struct kfd_process *p,
+ struct kfd_ioctl_pmc_settings *args)
+{
+ struct kfd_process_device *pdd;
+ struct device_queue_manager *dqm;
+ int status;
+
+ /* Check if we have the correct permissions. */
+ if (!perfmon_capable())
+ return -EPERM;
+
+ /* Lock/Unlock the device based on the parameter given in OP */
+ status = profile_lock_device(p, args->gpu_id, args->lock);
+ if (status != 0)
+ return status;
+
+ /* Enable/disable perfcount if requested */
+ mutex_lock(&p->mutex);
+ pdd = kfd_process_device_data_by_id(p, args->gpu_id);
+ dqm = pdd->dev->dqm;
+ mutex_unlock(&p->mutex);
+
+ dqm->ops.set_perfcount(dqm, args->perfcount_enable);
+ return status;
+}
+
+static int kfd_ioctl_profiler(struct file *filep, struct kfd_process *p, void
*data)
+{
+ struct kfd_ioctl_profiler_args *args = data;
+
+ switch (args->op) {
+ case KFD_IOC_PROFILER_VERSION:
+ args->version = KFD_IOC_PROFILER_VERSION_NUM;
+ return 0;
+ case KFD_IOC_PROFILER_PMC:
+ return kfd_profiler_pmc(p, &args->pmc);
+ }
+ return -EINVAL;
+}
+
#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
.cmd_drv = 0, .name = #ioctl}
@@ -3325,6 +3404,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_PROCESS,
kfd_ioctl_create_process, 0),
+
+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_PROFILER,
+ kfd_ioctl_profiler, 0),
};
#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 9a66ee661e57..f231e46e8528 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -936,6 +936,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
svm_range_set_max_pages(kfd->adev);
+ kfd->profiler_process = NULL;
+ mutex_init(&kfd->profiler_lock);
+
kfd->init_complete = true;
dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
kfd->adev->pdev->device);
@@ -971,6 +974,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
ida_destroy(&kfd->doorbell_ida);
kfd_gtt_sa_fini(kfd);
amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem);
+ mutex_destroy(&kfd->profiler_lock);
}
kfree(kfd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 804851632c4c..4170a283db5b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -305,6 +305,29 @@ static int remove_queue_mes(struct device_queue_manager
*dqm, struct queue *q,
return r;
}
+static void set_perfcount(struct device_queue_manager *dqm, int enable)
+{
+ struct device_process_node *cur;
+ struct qcm_process_device *qpd;
+ struct queue *q;
+ struct mqd_update_info minfo = { 0 };
+
+ if (!dqm)
+ return;
+
+ minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE :
+ UPDATE_FLAG_PERFCOUNT_DISABLE);
+ dqm_lock(dqm);
+ list_for_each_entry(cur, &dqm->queues, list) {
+ qpd = cur->qpd;
+ list_for_each_entry(q, &qpd->queues_list, list) {
+ pqm_update_mqd(qpd->pqm, q->properties.queue_id,
+ &minfo);
+ }
+ }
+ dqm_unlock(dqm);
+}
+
static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
{
struct device_process_node *cur;
@@ -2967,6 +2990,7 @@ struct device_queue_manager
*device_queue_manager_init(struct kfd_node *dev)
dqm->ops.reset_queues = reset_queues_cpsch;
dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
dqm->ops.checkpoint_mqd = checkpoint_mqd;
+ dqm->ops.set_perfcount = set_perfcount;
break;
case KFD_SCHED_POLICY_NO_HWS:
/* initialize dqm for no cp scheduling */
@@ -2987,6 +3011,7 @@ struct device_queue_manager
*device_queue_manager_init(struct kfd_node *dev)
dqm->ops.get_wave_state = get_wave_state;
dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
dqm->ops.checkpoint_mqd = checkpoint_mqd;
+ dqm->ops.set_perfcount = set_perfcount;
break;
default:
dev_err(dev->adev->dev, "Invalid scheduling policy %d\n",
dqm->sched_policy);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index ef07e44916f8..74a3bcec3e4e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -200,6 +200,8 @@ struct device_queue_manager_ops {
const struct queue *q,
void *mqd,
void *ctl_stack);
+ void (*set_perfcount)(struct device_queue_manager *dqm,
+ int enable);
};
struct device_queue_manager_asic_ops {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 97055f808d4a..993d60a24d50 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -124,10 +124,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
*/
m->cp_hqd_hq_scheduler0 = 1 << 14;
- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- }
if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
@@ -142,6 +141,12 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -221,6 +226,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
if (mm->dev->kfd->cwsr_enabled)
m->cp_hqd_ctx_save_control = 0;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
+
update_cu_mask(mm, mqd, minfo);
set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index 7e5a7ab6d0c0..4a574bbb5f37 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -164,10 +164,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
m->cp_hqd_hq_status0 |= 1 << 29;
- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- }
if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
@@ -182,6 +181,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -259,6 +263,12 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
}
if (mm->dev->kfd->cwsr_enabled)
m->cp_hqd_ctx_save_control = 0;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
update_cu_mask(mm, mqd, minfo);
set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
index a51f217329db..7173f6470e5e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
@@ -139,10 +139,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
m->cp_hqd_hq_status0 |= 1 << 29;
- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- }
if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
@@ -157,6 +156,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index dcf4bbfa641b..d4659a438be5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -218,10 +218,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- if (q->tba_addr) {
+ if (q->tba_addr)
m->compute_pgm_rsrc2 |=
(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
- }
if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
m->cp_hqd_persistent_state |=
@@ -236,6 +235,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -318,6 +322,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
m->cp_hqd_ctx_save_control = 0;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
+
if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3) &&
KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 4) &&
KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 5, 0))
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 09483f0862d4..e8967f5e3892 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -149,6 +149,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -231,6 +236,12 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd,
m->cp_hqd_ctx_save_control =
atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
update_cu_mask(mm, mqd, minfo);
set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9849b54f54ba..8983065645fa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -384,6 +384,11 @@ struct kfd_dev {
int kfd_dev_lock;
atomic_t kfd_processes_count;
+
+ /* Lock for profiler process */
+ struct mutex profiler_lock;
+ /* Process currently holding the lock */
+ struct kfd_process *profiler_process;
};
enum kfd_mempool {
@@ -556,6 +561,8 @@ enum mqd_update_flag {
UPDATE_FLAG_DBG_WA_ENABLE = 1,
UPDATE_FLAG_DBG_WA_DISABLE = 2,
UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
+ UPDATE_FLAG_PERFCOUNT_ENABLE = 5,
+ UPDATE_FLAG_PERFCOUNT_DISABLE = 6,
};
struct mqd_update_info {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 8511fbebf327..deca19b478d0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1110,6 +1110,16 @@ static void kfd_process_free_outstanding_kfd_bos(struct
kfd_process *p)
kfd_process_device_free_bos(p->pdds[i]);
}
+static void kfd_process_profiler_release(struct kfd_process *p, struct kfd_process_device *pdd)
+{
+ mutex_lock(&pdd->dev->kfd->profiler_lock);
+ if (pdd->dev->kfd->profiler_process == p) {
+ pdd->qpd.dqm->ops.set_perfcount(pdd->qpd.dqm, 0);
+ pdd->dev->kfd->profiler_process = NULL;
+ }
+ mutex_unlock(&pdd->dev->kfd->profiler_lock);
+}
+
static void kfd_process_destroy_pdds(struct kfd_process *p)
{
int i;
@@ -1121,6 +1131,7 @@ static void kfd_process_destroy_pdds(struct kfd_process
*p)
pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
pdd->dev->id, p->lead_thread->pid);
+ kfd_process_profiler_release(p, pdd);
kfd_process_device_destroy_cwsr_dgpu(pdd);
kfd_process_device_destroy_ib_mem(pdd);
diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
index e72359370857..abb526c915c3 100644
--- a/include/uapi/linux/kfd_ioctl.h
+++ b/include/uapi/linux/kfd_ioctl.h
@@ -1558,6 +1558,30 @@ struct kfd_ioctl_dbg_trap_args {
};
};
+#define KFD_IOC_PROFILER_VERSION_NUM 1
+enum kfd_profiler_ops {
+ KFD_IOC_PROFILER_PMC = 0,
+ KFD_IOC_PROFILER_PC_SAMPLE = 1,
