+ start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+ 0, adev->kfd.dev->id, prange->prefetch_loc,
+ prange->preferred_loc, trigger, timestamp);
+
svm_range_dma_unmap(adev->dev, scratch, 0, npages);
svm_range_free_dma_mappings(prange);
@@ -466,10 +477,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
kvfree(buf);
out:
if (!r && cpages) {
- pdd = svm_range_get_pdd_by_adev(prange, adev);
- if (pdd)
- WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
-
+ WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
return cpages;
}
return r;
@@ -480,6 +488,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
* @prange: range structure
* @best_loc: the device to migrate to
* @mm: the process mm structure
+ * @trigger: reason of migration
*
* Context: Process context, caller hold mmap read lock, svms lock, prange
lock
*
@@ -488,7 +497,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct
svm_range *prange,
*/
static int
svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
- struct mm_struct *mm)
+ struct mm_struct *mm, uint32_t trigger)
{
unsigned long addr, start, end;
struct vm_area_struct *vma;
@@ -525,7 +534,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t
best_loc,
break;
next = min(vma->vm_end, end);
- r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next);
+ r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next,
trigger);
if (r < 0) {
pr_debug("failed %ld to migrate\n", r);
break;
@@ -641,12 +650,14 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev,
struct svm_range *prange,
static long
svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
- struct vm_area_struct *vma, uint64_t start, uint64_t end)
+ struct vm_area_struct *vma, uint64_t start, uint64_t end,
+ uint32_t trigger)
{
uint64_t npages = (end - start) >> PAGE_SHIFT;
+ uint64_t timestamp = ktime_get_boottime_ns();
unsigned long upages = npages;
unsigned long cpages = 0;
- struct kfd_process_device *pdd;
+ struct kfd_process_device *pdd = NULL;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate;
dma_addr_t *scratch;
@@ -671,6 +682,10 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct
svm_range *prange,
migrate.dst = migrate.src + npages;
scratch = (dma_addr_t *)(migrate.dst + npages);
+ pdd = svm_range_get_pdd_by_adev(prange, adev);
+ if (!pdd)
+ goto out_free;
+
r = migrate_vma_setup(&migrate);
if (r) {
dev_err(adev->dev, "vma setup fail %d range [0x%lx 0x%lx]\n", r,
@@ -701,16 +716,19 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct
svm_range *prange,
svm_migrate_copy_done(adev, mfence);
migrate_vma_finalize(&migrate);
+
+ kfd_smi_event_migration(adev->kfd.dev, pdd->process->pasid,
+ start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+ adev->kfd.dev->id, 0, prange->prefetch_loc,
+ prange->preferred_loc, trigger, timestamp);
+
svm_range_dma_unmap(adev->dev, scratch, 0, npages);
out_free:
kvfree(buf);
out:
if (!r && cpages) {
- pdd = svm_range_get_pdd_by_adev(prange, adev);
- if (pdd)
- WRITE_ONCE(pdd->page_out, pdd->page_out + cpages);
-
+ WRITE_ONCE(pdd->page_out, pdd->page_out + cpages);
return upages;
}
return r ? r : upages;
@@ -720,13 +738,15 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct
svm_range *prange,
* svm_migrate_vram_to_ram - migrate svm range from device to system
* @prange: range structure
* @mm: process mm, use current->mm if NULL
+ * @trigger: reason of migration
*
* Context: Process context, caller hold mmap read lock, svms lock, prange
lock
*
* Return:
* 0 - OK, otherwise error code
*/
-int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm)
+int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
+ uint32_t trigger)
{
struct amdgpu_device *adev;
struct vm_area_struct *vma;
@@ -764,7 +784,7 @@ int svm_migrate_vram_to_ram(struct svm_range *prange,
struct mm_struct *mm)
break;
next = min(vma->vm_end, end);
- r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next);
+ r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next,
trigger);
if (r < 0) {
pr_debug("failed %ld to migrate\n", r);
break;
@@ -787,6 +807,7 @@ int svm_migrate_vram_to_ram(struct svm_range *prange,
struct mm_struct *mm)
* @prange: range structure
* @best_loc: the device to migrate to
* @mm: process mm, use current->mm if NULL
+ * @trigger: reason of migration
*
* Context: Process context, caller hold mmap read lock, svms lock, prange
lock
*
@@ -795,7 +816,7 @@ int svm_migrate_vram_to_ram(struct svm_range *prange,
struct mm_struct *mm)
*/
static int
svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
- struct mm_struct *mm)
+ struct mm_struct *mm, uint32_t trigger)
{
int r, retries = 3;
@@ -807,7 +828,7 @@ svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
do {
- r = svm_migrate_vram_to_ram(prange, mm);
+ r = svm_migrate_vram_to_ram(prange, mm, trigger);
if (r)
return r;
} while (prange->actual_loc && --retries);
@@ -815,17 +836,17 @@ svm_migrate_vram_to_vram(struct svm_range *prange,
uint32_t best_loc,
if (prange->actual_loc)
return -EDEADLK;
- return svm_migrate_ram_to_vram(prange, best_loc, mm);
+ return svm_migrate_ram_to_vram(prange, best_loc, mm, trigger);
}
int
svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
- struct mm_struct *mm)
+ struct mm_struct *mm, uint32_t trigger)
{
if (!prange->actual_loc)
- return svm_migrate_ram_to_vram(prange, best_loc, mm);
+ return svm_migrate_ram_to_vram(prange, best_loc, mm, trigger);
else
- return svm_migrate_vram_to_vram(prange, best_loc, mm);
+ return svm_migrate_vram_to_vram(prange, best_loc, mm, trigger);
}
@@ -894,7 +915,7 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
goto out_unlock_prange;
}
- r = svm_migrate_vram_to_ram(prange, mm);
+ r = svm_migrate_vram_to_ram(prange, mm,
MIGRATION_TRIGGER_PAGEFAULT_CPU);
if (r)
pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,
prange, prange->start, prange->last);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
index 2f5b3394c9ed..b3f0754b32fa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
@@ -41,8 +41,9 @@ enum MIGRATION_COPY_DIR {
};
int svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
- struct mm_struct *mm);
-int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm);
+ struct mm_struct *mm, uint32_t trigger);
+int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
+ uint32_t trigger);
unsigned long
svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
index 080eba0d3be0..97393f4f3549 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
@@ -314,6 +314,35 @@ void kfd_smi_event_page_fault_end(struct kfd_dev *dev,
pid_t pid,
len);
}
+void kfd_smi_event_migration(struct kfd_dev *dev, uint16_t pasid,
+ unsigned long start, unsigned long end,
+ uint32_t from, uint32_t to,
+ uint32_t prefetch_loc, uint32_t preferred_loc,
+ uint32_t trigger, uint64_t ts)
+{
+ struct amdgpu_task_info task_info = {0};
+ char fifo_in[128];
+ uint64_t duration;
+ pid_t pid;
+ int len;
+
+ if (!kfd_smi_event_duration(dev, ts, &duration))
+ return;
+
+ amdgpu_vm_get_task_info(dev->adev, pasid, &task_info);
+ if (!task_info.pid)
+ return;
+ pid = task_info.pid;
+
+ len = snprintf(fifo_in, sizeof(fifo_in),
+ "%x %lld(%lld) -%d @%lx(%lx) %x->%x %x:%x %d\n",
+ KFD_SMI_EVENT_MIGRATION, ts, duration, pid, start,
+ end - start, from, to, prefetch_loc, preferred_loc,
+ trigger);
+
+ add_event_to_kfifo(pid, dev, KFD_SMI_EVENT_MIGRATION, fifo_in, len);
+}
+
int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd)
{
struct kfd_smi_client *client;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
index 7f70db914d2c..5788728f2879 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
@@ -34,4 +34,9 @@ void kfd_smi_event_page_fault_start(struct kfd_dev *dev,
pid_t pid,
void kfd_smi_event_page_fault_end(struct kfd_dev *dev, pid_t pid,
unsigned long address, bool migration,
uint64_t ts);
+void kfd_smi_event_migration(struct kfd_dev *dev, uint16_t pasid,
+ unsigned long start, unsigned long end,
+ uint32_t from, uint32_t to,
+ uint32_t prefetch_loc, uint32_t preferred_loc,
+ uint32_t trigger, uint64_t ts);
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 08b21f9759ea..7dbc724364e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -2791,7 +2791,8 @@ svm_range_restore_pages(struct amdgpu_device *adev,
unsigned int pasid,
if (prange->actual_loc != best_loc) {
migration = true;
if (best_loc) {
- r = svm_migrate_to_vram(prange, best_loc, mm);
+ r = svm_migrate_to_vram(prange, best_loc, mm,
+ MIGRATION_TRIGGER_PAGEFAULT);
if (r) {
pr_debug("svm_migrate_to_vram failed (%d) at %llx,
falling back to system memory\n",
r, addr);
@@ -2799,12 +2800,14 @@ svm_range_restore_pages(struct amdgpu_device *adev,
unsigned int pasid,
* VRAM failed
*/
if (prange->actual_loc)
- r = svm_migrate_vram_to_ram(prange, mm);
+ r = svm_migrate_vram_to_ram(prange, mm,
+ MIGRATION_TRIGGER_PAGEFAULT);
else
r = 0;
}
} else {
- r = svm_migrate_vram_to_ram(prange, mm);
+ r = svm_migrate_vram_to_ram(prange, mm,
+
MIGRATION_TRIGGER_PAGEFAULT);
}
if (r) {
pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n",
@@ -3126,12 +3129,12 @@ svm_range_trigger_migration(struct mm_struct *mm,
struct svm_range *prange,
return 0;
if (!best_loc) {
- r = svm_migrate_vram_to_ram(prange, mm);
+ r = svm_migrate_vram_to_ram(prange, mm,
MIGRATION_TRIGGER_PREFETCH);
*migrated = !r;
return r;
}
- r = svm_migrate_to_vram(prange, best_loc, mm);
+ r = svm_migrate_to_vram(prange, best_loc, mm,
MIGRATION_TRIGGER_PREFETCH);
*migrated = !r;
return r;
@@ -3188,7 +3191,8 @@ static void svm_range_evict_svm_bo_worker(struct
work_struct *work)
mutex_lock(&prange->migrate_mutex);
do {
svm_migrate_vram_to_ram(prange,
- svm_bo->eviction_fence->mm);
+ svm_bo->eviction_fence->mm,
+ MIGRATION_TRIGGER_TTM_EVICTION);
} while (prange->actual_loc && --retries);
WARN(prange->actual_loc, "Migration failed during eviction");
