Re: [PATCH v2 5/8] drm/amdkfd: add migration SMI event
Am 2022-01-20 um 18:13 schrieb Philip Yang:
After migration is finished, output timestamp when migration starts,
duration of migration, svm range address and size, GPU id of
migration source and destination and svm range attributes,
Migration trigger could be prefetch, CPU or GPU page fault and TTM
eviction.
Signed-off-by: Philip Yang
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c| 67 ++---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.h| 5 +-
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c | 29 +
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h | 5 ++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 16 +++--
5 files changed, 91 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 88db82b3d443..06fb888f87aa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -32,6 +32,7 @@
#include "kfd_priv.h"
#include "kfd_svm.h"
#include "kfd_migrate.h"
+#include "kfd_smi_events.h"
#ifdef dev_fmt
#undef dev_fmt
@@ -402,10 +403,11 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev,
struct svm_range *prange,
static long
svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
struct vm_area_struct *vma, uint64_t start,
- uint64_t end)
+ uint64_t end, uint32_t trigger)
{
uint64_t npages = (end - start) >> PAGE_SHIFT;
- struct kfd_process_device *pdd;
+ struct kfd_process_device *pdd = NULL;
+ uint64_t timestamp = ktime_get_boottime_ns();
struct dma_fence *mfence = NULL;
struct migrate_vma migrate;
unsigned long cpages = 0;
@@ -431,6 +433,10 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
migrate.dst = migrate.src + npages;
scratch = (dma_addr_t *)(migrate.dst + npages);
+ pdd = svm_range_get_pdd_by_adev(prange, adev);
+ if (!pdd)
+ goto out_free;
+
r = migrate_vma_setup();
if (r) {
dev_err(adev->dev, "vma setup fail %d range [0x%lx 0x%lx]\n", r,
@@ -459,6 +465,11 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
svm_migrate_copy_done(adev, mfence);
migrate_vma_finalize();
+ kfd_smi_event_migration(adev->kfd.dev, pdd->process->pasid,
The pasid is only used by kfd_smi_event_migration to determine the PID.
But you already know the PID here:
container_of(prange->svms, struct kfd_process, svms)->lead_thread->pid;
+ start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+ 0, adev->kfd.dev->id, prange->prefetch_loc,
+ prange->preferred_loc, trigger, timestamp);
+
svm_range_dma_unmap(adev->dev, scratch, 0, npages);
svm_range_free_dma_mappings(prange);
@@ -466,10 +477,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
kvfree(buf);
out:
if (!r && cpages) {
- pdd = svm_range_get_pdd_by_adev(prange, adev);
- if (pdd)
- WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
-
+ WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
return cpages;
}
return r;
@@ -480,6 +488,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
* @prange: range structure
* @best_loc: the device to migrate to
* @mm: the process mm structure
+ * @trigger: reason of migration
*
* Context: Process context, caller hold mmap read lock, svms lock, prange
lock
*
@@ -488,7 +497,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
*/
static int
svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
- struct mm_struct *mm)
+ struct mm_struct *mm, uint32_t trigger)
{
unsigned long addr, start, end;
struct vm_area_struct *vma;
@@ -525,7 +534,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t
best_loc,
break;
next = min(vma->vm_end, end);
- r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next);
+ r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next,
trigger);
if (r < 0) {
pr_debug("failed %ld to migrate\n", r);
break;
@@ -641,12 +650,14 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev,
struct svm_range *prange,
static long
svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
- struct vm_area_struct *vma, uint64_t start, uint64_t end)
+ struct vm_area_struct *vma, uint64_t start, uint64_t end,
+ uint32_t trigger)
{
uint64_t npages =
[PATCH v2 5/8] drm/amdkfd: add migration SMI event
After migration is finished, output timestamp when migration starts,
duration of migration, svm range address and size, GPU id of
migration source and destination and svm range attributes,
Migration trigger could be prefetch, CPU or GPU page fault and TTM
eviction.
Signed-off-by: Philip Yang
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c| 67 ++---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.h| 5 +-
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c | 29 +
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h | 5 ++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 16 +++--
5 files changed, 91 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 88db82b3d443..06fb888f87aa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -32,6 +32,7 @@
#include "kfd_priv.h"
#include "kfd_svm.h"
#include "kfd_migrate.h"
+#include "kfd_smi_events.h"
#ifdef dev_fmt
#undef dev_fmt
@@ -402,10 +403,11 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev,
struct svm_range *prange,
static long
svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
struct vm_area_struct *vma, uint64_t start,
- uint64_t end)
+ uint64_t end, uint32_t trigger)
{
uint64_t npages = (end - start) >> PAGE_SHIFT;
- struct kfd_process_device *pdd;
+ struct kfd_process_device *pdd = NULL;
+ uint64_t timestamp = ktime_get_boottime_ns();
struct dma_fence *mfence = NULL;
struct migrate_vma migrate;
unsigned long cpages = 0;
@@ -431,6 +433,10 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
migrate.dst = migrate.src + npages;
scratch = (dma_addr_t *)(migrate.dst + npages);
+ pdd = svm_range_get_pdd_by_adev(prange, adev);
+ if (!pdd)
+ goto out_free;
+
r = migrate_vma_setup();
if (r) {
dev_err(adev->dev, "vma setup fail %d range [0x%lx 0x%lx]\n", r,
@@ -459,6 +465,11 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
svm_migrate_copy_done(adev, mfence);
migrate_vma_finalize();
+ kfd_smi_event_migration(adev->kfd.dev, pdd->process->pasid,
+ start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+ 0, adev->kfd.dev->id, prange->prefetch_loc,
+ prange->preferred_loc, trigger, timestamp);
+
svm_range_dma_unmap(adev->dev, scratch, 0, npages);
svm_range_free_dma_mappings(prange);
@@ -466,10 +477,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
kvfree(buf);
out:
if (!r && cpages) {
- pdd = svm_range_get_pdd_by_adev(prange, adev);
- if (pdd)
- WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
-
+ WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
return cpages;
}
return r;
@@ -480,6 +488,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
* @prange: range structure
* @best_loc: the device to migrate to
* @mm: the process mm structure
+ * @trigger: reason of migration
*
* Context: Process context, caller hold mmap read lock, svms lock, prange lock
*
@@ -488,7 +497,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
*/
static int
svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
- struct mm_struct *mm)
+ struct mm_struct *mm, uint32_t trigger)
{
unsigned long addr, start, end;
struct vm_area_struct *vma;
@@ -525,7 +534,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t
best_loc,
break;
next = min(vma->vm_end, end);
- r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next);
+ r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next,
trigger);
if (r < 0) {
pr_debug("failed %ld to migrate\n", r);
break;
@@ -641,12 +650,14 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev,
struct svm_range *prange,
static long
svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
- struct vm_area_struct *vma, uint64_t start, uint64_t end)
+ struct vm_area_struct *vma, uint64_t start, uint64_t end,
+ uint32_t trigger)
{
uint64_t npages = (end - start) >> PAGE_SHIFT;
+ uint64_t timestamp = ktime_get_boottime_ns();
unsigned long upages = npages;
unsigned long cpages = 0;
- struct kfd_process_device *pdd;
+ struct kfd_process_device *pdd = NULL;
struct
