Replace the mapped_to_gpu boolean with bitmap_mapped to track which
GPUs currently have the range mapped. Set bits in svm_range_map_to_gpus()
and clear them in svm_range_unmap_from_gpus(). This is separate from
bitmap_access/bitmap_aip which track user-requested attributes and must
not be used to determine mapping state.
Add bitmap_needs_unmap to svm_range, set when a GPU is given no-access.
Add svm_range_needs_unmap() to trigger the unmap once bitmap_needs_unmap
equals bitmap_mapped, i.e. all mapped GPUs are set to no-access.
v4:
- Rename and set prange->mapping_done to false if validate and map not
complete successfully (Felix)
v3:
- Correct error handling, support app retry update mapping (Felix)
v2:
- Add bitmap_mapped to not break get_attr (Felix)
Signed-off-by: Philip Yang<[email protected]>
Reviewed-by: Felix Kuehling<[email protected]>
---
drivers/gpu/drm/amd/amdkfd/kfd_queue.c | 5 +-
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 103 ++++++++++++++++---------
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 7 +-
3 files changed, 73 insertions(+), 42 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 9d4838461168..5d55407069a4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -111,12 +111,11 @@ static int kfd_queue_buffer_svm_get(struct
kfd_process_device *pdd, u64 addr, u6
if (!prange)
break;
- if (!prange->mapped_to_gpu)
- break;
-
r = kfd_process_gpuid_from_node(p, pdd->dev, &gpuid, &gpuidx);
if (r < 0)
break;
+ if (!test_bit(gpuidx, prange->bitmap_mapped))
+ break;
if (!test_bit(gpuidx, prange->bitmap_access) &&
!test_bit(gpuidx, prange->bitmap_aip))
break;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index df7fca65e9a2..e039b6f2942f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -786,6 +786,7 @@ svm_range_apply_attrs(struct kfd_process *p, struct
svm_range *prange,
if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {
bitmap_clear(prange->bitmap_access, gpuidx, 1);
bitmap_clear(prange->bitmap_aip, gpuidx, 1);
+ bitmap_set(prange->bitmap_needs_unmap, gpuidx,
1);
} else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {
bitmap_set(prange->bitmap_access, gpuidx, 1);
bitmap_clear(prange->bitmap_aip, gpuidx, 1);
@@ -1076,9 +1077,10 @@ svm_range_split_adjust(struct svm_range *new, struct
svm_range *old,
new->prefetch_loc = old->prefetch_loc;
new->actual_loc = old->actual_loc;
new->granularity = old->granularity;
- new->mapped_to_gpu = old->mapped_to_gpu;
+ new->mapping_done = old->mapping_done;
bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
+ bitmap_copy(new->bitmap_mapped, old->bitmap_mapped, MAX_GPU_INSTANCE);
atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount));
return 0;
@@ -1379,7 +1381,8 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
static int
svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
- unsigned long last, uint32_t trigger)
+ unsigned long last, unsigned long *bitmap_unmap,
+ uint32_t trigger)
{
struct kfd_process_device *pdd;
struct dma_fence *fence = NULL;
@@ -1387,21 +1390,15 @@ svm_range_unmap_from_gpus(struct svm_range *prange,
unsigned long start,
uint32_t gpuidx;
int r = 0;
- if (!prange->mapped_to_gpu) {
- pr_debug("prange 0x%p [0x%lx 0x%lx] not mapped to GPU\n",
- prange, prange->start, prange->last);
- return 0;
- }
-
- if (prange->start == start && prange->last == last) {
- pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange);
- prange->mapped_to_gpu = false;
- }
-
p = container_of(prange->svms, struct kfd_process, svms);
- for_each_or_bit(gpuidx, prange->bitmap_access, prange->bitmap_aip, MAX_GPU_INSTANCE) {
- pr_debug("unmap from gpu idx 0x%x\n", gpuidx);
+ for_each_set_bit(gpuidx, bitmap_unmap, MAX_GPU_INSTANCE) {
+ if (prange->start == start && prange->last == last) {
+ pr_debug("unmap svms 0x%p prange 0x%p from gpu_idx
0x%x\n",
+ prange->svms, prange, gpuidx);
+ clear_bit(gpuidx, prange->bitmap_mapped);
+ }
+
pdd = kfd_process_device_from_gpuidx(p, gpuidx);
if (!pdd) {
pr_debug("failed to find device idx %d\n", gpuidx);
@@ -1554,6 +1551,8 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned
long offset,
continue;
}
+ set_bit(gpuidx, prange->bitmap_mapped);
+
r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
prange->dma_addr[gpuidx],
bo_adev, wait ? &fence : NULL,
@@ -1699,7 +1698,9 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
bitmap_zero(ctx->bitmap, MAX_GPU_INSTANCE);
bitmap_set(ctx->bitmap, gpuidx, 1);
} else if (ctx->process->xnack_enabled) {
- bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
+ /* Update mapping on already mapped or access in place GPU */
+ bitmap_or(ctx->bitmap, prange->bitmap_mapped,
prange->bitmap_aip,
+ MAX_GPU_INSTANCE);
/* If prefetch range to GPU, or GPU retry fault migrate range to
* GPU, which has ACCESS attribute to the range, create mapping
@@ -1719,14 +1720,12 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
}
/*
- * If prange is already mapped or with always mapped flag,
- * update mapping on GPUs with ACCESS attribute
+ * If prange with always mapped flag, update mapping on GPUs
with
+ * ACCESS attribute
*/
- if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) {
- if (prange->mapped_to_gpu ||
- prange->flags &
KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
- bitmap_copy(ctx->bitmap, prange->bitmap_access,
MAX_GPU_INSTANCE);
- }
+ if (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
+ bitmap_or(ctx->bitmap, ctx->bitmap,
prange->bitmap_access,
+ MAX_GPU_INSTANCE);
} else {
bitmap_or(ctx->bitmap, prange->bitmap_access,
prange->bitmap_aip, MAX_GPU_INSTANCE);
@@ -1792,6 +1791,7 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
e = min(end, prange->last);
if (e >= s)
r = svm_range_unmap_from_gpus(prange,
s, e,
+ prange->bitmap_mapped,
KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU);
svm_range_unlock(prange);
/* If unmap returns non-zero, we'll bail on the
next for loop
@@ -1854,7 +1854,9 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
}
if (!r && next == end)
- prange->mapped_to_gpu = true;
+ prange->mapping_done = true;
+ else
+ prange->mapping_done = false;
svm_range_unlock(prange);
@@ -2024,10 +2026,10 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
if (!p->xnack_enabled ||
(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) {
int evicted_ranges;
- bool mapped = prange->mapped_to_gpu;
+ bool mapped = !bitmap_empty(prange->bitmap_mapped,
MAX_GPU_INSTANCE);
list_for_each_entry(pchild, &prange->child_list, child_list) {
- if (!pchild->mapped_to_gpu)
+ if (bitmap_empty(pchild->bitmap_mapped,
MAX_GPU_INSTANCE))
continue;
mapped = true;
mutex_lock_nested(&pchild->lock, 1);
@@ -2076,13 +2078,14 @@ svm_range_evict(struct svm_range *prange, struct
mm_struct *mm,
s = max(start, pchild->start);
l = min(last, pchild->last);
if (l >= s)
- svm_range_unmap_from_gpus(pchild, s, l,
trigger);
+ svm_range_unmap_from_gpus(pchild, s, l,
prange->bitmap_mapped,
+ trigger);
mutex_unlock(&pchild->lock);
}
s = max(start, prange->start);
l = min(last, prange->last);
if (l >= s)
- svm_range_unmap_from_gpus(prange, s, l, trigger);
+ svm_range_unmap_from_gpus(prange, s, l,
prange->bitmap_mapped, trigger);
}
return r;
@@ -2112,10 +2115,11 @@ static struct svm_range *svm_range_clone(struct
svm_range *old)
new->prefetch_loc = old->prefetch_loc;
new->actual_loc = old->actual_loc;
new->granularity = old->granularity;
- new->mapped_to_gpu = old->mapped_to_gpu;
+ new->mapping_done = old->mapping_done;
new->vram_pages = old->vram_pages;
bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
+ bitmap_copy(new->bitmap_mapped, old->bitmap_mapped, MAX_GPU_INSTANCE);
atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount));
return new;
@@ -2235,7 +2239,7 @@ svm_range_add(struct kfd_process *p, uint64_t start,
uint64_t size,
next_start = min(node->last, last) + 1;
if (svm_range_is_same_attrs(p, prange, nattr, attrs) &&
- prange->mapped_to_gpu) {
+ prange->mapping_done) {
/* nothing to do */
} else if (node->start < start || node->last > last) {
/* node intersects the update range and its attributes
@@ -2616,14 +2620,14 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct
svm_range *prange,
s = max(start, pchild->start);
l = min(last, pchild->last);
if (l >= s)
- svm_range_unmap_from_gpus(pchild, s, l, trigger);
+ svm_range_unmap_from_gpus(pchild, s, l,
prange->bitmap_mapped, trigger);
svm_range_unmap_split(prange, pchild, start, last);
mutex_unlock(&pchild->lock);
}
s = max(start, prange->start);
l = min(last, prange->last);
if (l >= s)
- svm_range_unmap_from_gpus(prange, s, l, trigger);
+ svm_range_unmap_from_gpus(prange, s, l, prange->bitmap_mapped,
trigger);
svm_range_unmap_split(prange, prange, start, last);
if (unmap_parent)
@@ -3706,6 +3710,23 @@ static void svm_range_evict_svm_bo_worker(struct
work_struct *work)
svm_range_bo_unref(svm_bo);
}
+static bool svm_range_needs_unmap(struct kfd_process *p, struct svm_range *prange)
+{
+ if (bitmap_empty(prange->bitmap_needs_unmap, MAX_GPU_INSTANCE))
+ return false;
+
+ pr_debug("prange 0x%p no access set for [0x%lx 0x%lx]\n",
+ prange, prange->start, prange->last);
+
+ svm_range_unmap_from_gpus(prange, prange->start,
+ prange->last, prange->bitmap_needs_unmap,
+ KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU);
+
+ bitmap_clear(prange->bitmap_needs_unmap, 0, MAX_GPU_INSTANCE);
+
+ return bitmap_empty(prange->bitmap_mapped, MAX_GPU_INSTANCE);
+}
+
static int
svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
uint64_t start, uint64_t size, uint32_t nattr,
@@ -3761,10 +3782,10 @@ svm_range_set_attr(struct kfd_process *p, struct
mm_struct *mm,
svm_range_add_to_svms(prange);
svm_range_add_notifier_locked(mm, prange);
}
- list_for_each_entry(prange, &update_list, update_list) {
+
+ list_for_each_entry(prange, &update_list, update_list)
svm_range_apply_attrs(p, prange, nattr, attrs, &update_mapping);
- /* TODO: unmap ranges from GPU that lost access */
- }
+
update_mapping |= !p->xnack_enabled && !list_empty(&remap_list);
list_for_each_entry_safe(prange, next, &remove_list, update_list) {
@@ -3785,6 +3806,9 @@ svm_range_set_attr(struct kfd_process *p, struct
mm_struct *mm,
list_for_each_entry(prange, &update_list, update_list) {
bool migrated;
+ if (svm_range_needs_unmap(p, prange))
+ continue;
+
mutex_lock(&prange->migrate_mutex);
r = svm_range_trigger_migration(mm, prange, &migrated);
@@ -3793,7 +3817,7 @@ svm_range_set_attr(struct kfd_process *p, struct
mm_struct *mm,
if (migrated && (!p->xnack_enabled ||
(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) &&
- prange->mapped_to_gpu) {
+ !bitmap_empty(prange->bitmap_mapped, MAX_GPU_INSTANCE)) {
pr_debug("restore_work will update mappings of GPUs\n");
mutex_unlock(&prange->migrate_mutex);
continue;
@@ -3804,7 +3828,8 @@ svm_range_set_attr(struct kfd_process *p, struct
mm_struct *mm,
continue;
}
- flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu;
+ flush_tlb = !migrated && update_mapping &&
+ !bitmap_empty(prange->bitmap_mapped,
MAX_GPU_INSTANCE);
r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,
MAX_GPU_INSTANCE, true, true,
flush_tlb);
@@ -3818,11 +3843,13 @@ svm_range_set_attr(struct kfd_process *p, struct
mm_struct *mm,
}
list_for_each_entry(prange, &remap_list, update_list) {
+ flush_tlb = !bitmap_empty(prange->bitmap_mapped,
MAX_GPU_INSTANCE);
+
pr_debug("Remapping prange 0x%p [0x%lx 0x%lx]\n",
prange, prange->start, prange->last);
mutex_lock(&prange->migrate_mutex);
r = svm_range_validate_and_map(mm, prange->start,
prange->last, prange,
- MAX_GPU_INSTANCE, true, true,
prange->mapped_to_gpu);
+ MAX_GPU_INSTANCE, true, true,
flush_tlb);
if (r)
pr_debug("failed %d on remap svm range\n", r);
mutex_unlock(&prange->migrate_mutex);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index a63dfc95b602..0da635532aff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -100,6 +100,9 @@ struct svm_work_list_item {
* @child_list: list header for split ranges which are not added to svms yet
* @bitmap_access: index bitmap of GPUs which can access the range
* @bitmap_aip: index bitmap of GPUs which can access the range in place
+ * @bitmap_needs_unmap: index bitmap of GPUs which currently set NO_ACCESS
+ * @bitmap_mapped: index bitmap of GPUs which currently have the range mapped
+ * @mapping_done: true if range_validate_and_map complete successfully
*
* Data structure for virtual memory range shared by CPU and GPUs, it can be
* allocated from system memory ram or device vram, and migrate from ram to
vram
@@ -135,7 +138,9 @@ struct svm_range {
struct list_head child_list;
DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
- bool mapped_to_gpu;
+ DECLARE_BITMAP(bitmap_needs_unmap, MAX_GPU_INSTANCE);
+ DECLARE_BITMAP(bitmap_mapped, MAX_GPU_INSTANCE);
+ bool mapping_done;
atomic_t queue_refcount;
};
