Embed VRAM migration directly into the per-range map loop. Introduce enum amdgpu_svm_migrate_mode to carry migration intent through the map_attr_ranges -> map_interval -> map call chain:
MIGRATE_PREFERRED - follow preferred_loc MIGRATE_TO_VRAM - one-shot prefetch, force VRAM migration MIGRATE_TO_SYSMEM - evict VRAM pages to system memory MIGRATE_NONE - suppress migration (restore worker) Inside per-range's map loop, call amdgpu_svm_range_migrate_range() which handles both migration directions on each drm_gpusvm_range: - TO_VRAM: migrate to VRAM via drm_pagemap_populate_mm() - TO_SYSMEM: evict VRAM via drm_gpusvm_range_evict() Signed-off-by: Junhua Shen <[email protected]> --- drivers/gpu/drm/amd/amdgpu/Makefile | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 143 +++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h | 5 +- .../drm/amd/amdgpu/amdgpu_svm_range_migrate.c | 195 ++++++++++++++++++ .../drm/amd/amdgpu/amdgpu_svm_range_migrate.h | 53 +++++ 6 files changed, 338 insertions(+), 68 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.h diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index e64abb5c8ab8..cf4f453b7e68 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -323,14 +323,14 @@ amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_hmm.o # svm support amdgpu-$(CONFIG_DRM_AMDGPU_SVM) += amdgpu_svm.o amdgpu_svm_attr.o \ - amdgpu_svm_range.o amdgpu_migrate.o + amdgpu_svm_range.o amdgpu_svm_range_migrate.o amdgpu_migrate.o .PHONY: clean-svm clean-svm: rm -f $(obj)/amdgpu_svm.o $(obj)/amdgpu_svm_attr.o $(obj)/amdgpu_svm_range.o \ - $(obj)/amdgpu_migrate.o \ + $(obj)/amdgpu_svm_range_migrate.o $(obj)/amdgpu_migrate.o \ $(obj)/.amdgpu_svm.o.cmd $(obj)/.amdgpu_svm_attr.o.cmd $(obj)/.amdgpu_svm_range.o.cmd \ - $(obj)/.amdgpu_migrate.o.cmd + $(obj)/.amdgpu_svm_range_migrate.o.cmd $(obj)/.amdgpu_migrate.o.cmd include $(FULL_AMD_PATH)/pm/Makefile diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c index 05a94a790e79..0310fda2b061 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c @@ -32,6 +32,7 @@ #include "amdgpu_svm.h" #include "amdgpu_svm_attr.h" #include "amdgpu_svm_range.h" +#include "amdgpu_svm_range_migrate.h" #include "amdgpu_vm.h" #if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM) @@ -342,7 +343,8 @@ int amdgpu_svm_handle_fault(struct amdgpu_device *adev, uint32_t pasid, AMDGPU_SVM_TRACE("handle_fault: map_attr page=0x%lx\n", fault_page); down_write(&svm->svm_lock); - ret = amdgpu_svm_range_map_attr_ranges(svm, fault_page, fault_page); + ret = amdgpu_svm_range_map_attr_ranges(svm, fault_page, fault_page, + AMDGPU_SVM_MIGRATE_PREFERRED); up_write(&svm->svm_lock); if (ret) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index 472a641fb836..0aa147df621a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -25,6 +25,7 @@ #include "amdgpu_svm.h" #include "amdgpu_svm_attr.h" #include "amdgpu_svm_range.h" +#include "amdgpu_migrate.h" #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "amdgpu_vm.h" @@ -75,6 +76,13 @@ amdgpu_svm_range_enqueue(struct amdgpu_svm *svm, unsigned long last, enum amdgpu_svm_range_queue_op op); +static bool +prefetch_loc_is_vram(const struct amdgpu_svm_attrs *attrs) +{ + return attrs->prefetch_loc != AMDGPU_SVM_LOCATION_SYSMEM && + attrs->prefetch_loc != AMDGPU_SVM_LOCATION_UNDEFINED; +} + static inline bool range_has_access(enum amdgpu_svm_attr_access access) { @@ -114,7 +122,6 @@ range_pages_valid(struct amdgpu_svm *svm, return drm_gpusvm_range_pages_valid(&svm->gpusvm, range); } - static int amdgpu_svm_range_gpu_unmap_in_notifier(struct amdgpu_svm *svm, struct drm_gpusvm_range *range, @@ -247,11 +254,11 @@ amdgpu_svm_range_attr_pte_flags(struct amdgpu_svm *svm, return pte_flags; } - /* - * POC/WA: reuse kfd apis for queue quiesce/resume - * But kfd apis are for process level, not for GPU VM level - * need consider potential issues - */ +/* + * POC/WA: reuse kfd apis for queue quiesce/resume + * But kfd apis are for process level, not for GPU VM level + * need consider potential issues + */ void amdgpu_svm_range_restore_begin_compute(struct amdgpu_svm *svm) { int ret; @@ -317,39 +324,6 @@ static int amdgpu_svm_range_lock_vm_pd(struct amdgpu_svm *svm, struct drm_exec * return 0; } -static int -amdgpu_svm_range_update_gpu(struct amdgpu_svm *svm, unsigned long start_page, - unsigned long last_page, uint64_t pte_flags, - dma_addr_t *pages_addr, bool flush_tlb, - bool update_pdes, bool wait_fence) -{ - struct drm_exec exec; - struct dma_fence *fence = NULL; - int ret; - - ret = amdgpu_svm_range_lock_vm_pd(svm, &exec); - if (ret) - return ret; - - ret = amdgpu_vm_update_range(svm->adev, svm->vm, false, false, - flush_tlb, true, - NULL, start_page, last_page, pte_flags, 0, 0, - NULL, pages_addr, wait_fence ? &fence : NULL); - if (!ret && wait_fence && fence) { - ret = dma_fence_wait(fence, false); - if (ret < 0) - AMDGPU_SVM_TRACE("wait unmap fence failed: ret=%d [0x%lx-0x%lx]-0x%lx\n", - ret, start_page, last_page, - last_page - start_page + 1); - } - if (!ret && update_pdes) - ret = amdgpu_vm_update_pdes(svm->adev, svm->vm, false); - - dma_fence_put(fence); - drm_exec_fini(&exec); - return ret; -} - static int amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm, struct drm_gpusvm_range *range, @@ -376,9 +350,11 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm, npages - mapped_pages); dma_addr_t seg_addr = entry->addr; unsigned long start_page, last_page; + uint64_t seg_pte_flags; bool is_last_seg; - if (entry->proto != DRM_INTERCONNECT_SYSTEM) + if (entry->proto != DRM_INTERCONNECT_SYSTEM && + entry->proto != AMDGPU_INTERCONNECT_VRAM) return -EOPNOTSUPP; while (mapped_pages + seg_pages < npages) { @@ -399,9 +375,20 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm, last_page = start_page + seg_pages - 1; is_last_seg = mapped_pages + seg_pages == npages; + /* + * For VRAM pages, device_map() already returned the MC + * address in seg_addr. Clear the SYSTEM and SNOOPED bits + * since this is local VRAM, not DMA-mapped system memory. + */ + if (entry->proto == AMDGPU_INTERCONNECT_VRAM) + seg_pte_flags = pte_flags & ~(AMDGPU_PTE_SYSTEM | + AMDGPU_PTE_SNOOPED); + else + seg_pte_flags = pte_flags; + ret = amdgpu_vm_update_range(svm->adev, svm->vm, false, false, flush_tlb && is_last_seg, true, NULL, - start_page, last_page, pte_flags, + start_page, last_page, seg_pte_flags, 0, seg_addr, NULL, NULL, wait_fence && is_last_seg ? fence : NULL); if (ret) @@ -419,7 +406,8 @@ amdgpu_svm_range_map(struct amdgpu_svm *svm, unsigned long end, const struct amdgpu_svm_attrs *attrs, const struct drm_gpusvm_ctx *gpusvm_ctx, - uint64_t pte_flags) + uint64_t pte_flags, + enum amdgpu_svm_migrate_mode migrate_mode) { unsigned long addr = start; int ret; @@ -462,6 +450,10 @@ amdgpu_svm_range_map(struct amdgpu_svm *svm, if (next_addr <= addr) return -EINVAL; + /* Per-range migration */ + amdgpu_svm_range_migrate_range(svm, range, attrs, + migrate_mode); + range_pte_flags = map_ctx.read_only ? (pte_flags & ~AMDGPU_PTE_WRITEABLE) : pte_flags; @@ -529,10 +521,16 @@ amdgpu_svm_range_map(struct amdgpu_svm *svm, static int amdgpu_svm_range_map_interval(struct amdgpu_svm *svm, unsigned long start_page, unsigned long last_page, - const struct amdgpu_svm_attrs *attrs) + const struct amdgpu_svm_attrs *attrs, + enum amdgpu_svm_migrate_mode migrate_mode) { + bool hw_devmem = amdgpu_pagemap_capable(svm); + struct drm_gpusvm_ctx gpusvm_ctx = { .read_only = !!(attrs->flags & AMDGPU_SVM_FLAG_GPU_RO), + .devmem_possible = hw_devmem, + .device_private_page_owner = hw_devmem ? + AMDGPU_SVM_PGMAP_OWNER(svm->adev) : NULL, }; unsigned long start = start_page << PAGE_SHIFT; unsigned long end = (last_page + 1) << PAGE_SHIFT; @@ -542,7 +540,7 @@ amdgpu_svm_range_map_interval(struct amdgpu_svm *svm, unsigned long start_page, pte_flags = amdgpu_svm_range_attr_pte_flags(svm, attrs); ret = amdgpu_svm_range_map(svm, start, end, attrs, &gpusvm_ctx, - pte_flags); + pte_flags, migrate_mode); if (ret) AMDGPU_SVM_TRACE("map_interval failed: ret=%d [0x%lx-0x%lx)-0x%lx\n", ret, start, end, end - start); @@ -553,7 +551,8 @@ amdgpu_svm_range_map_interval(struct amdgpu_svm *svm, unsigned long start_page, int amdgpu_svm_range_map_attr_ranges(struct amdgpu_svm *svm, unsigned long start_page, - unsigned long last_page) + unsigned long last_page, + enum amdgpu_svm_migrate_mode migrate_mode) { lockdep_assert_held_write(&svm->svm_lock); @@ -573,9 +572,10 @@ amdgpu_svm_range_map_attr_ranges(struct amdgpu_svm *svm, seg_last = min(seg_last, last_page); if (range_has_access(attrs.access)) { - /* map may fail here cause no vma or access deny */ - ret = amdgpu_svm_range_map_interval(svm, cursor, seg_last, - &attrs); + ret = amdgpu_svm_range_map_interval(svm, cursor, + seg_last, + &attrs, + migrate_mode); if (ret) return ret; } @@ -657,7 +657,6 @@ static int amdgpu_svm_range_rebuild_locked(struct amdgpu_svm *svm, unsigned long rebuild_start = start_page; unsigned long rebuild_last = last_page; bool removed; - int ret; lockdep_assert_held_write(&svm->svm_lock); @@ -673,14 +672,10 @@ static int amdgpu_svm_range_rebuild_locked(struct amdgpu_svm *svm, /* scan rebuild start end to build the extra removed ranges */ if (rebuild) return amdgpu_svm_range_map_attr_ranges(svm, rebuild_start, - rebuild_last); + rebuild_last, + AMDGPU_SVM_MIGRATE_PREFERRED); - ret = amdgpu_svm_range_update_gpu(svm, rebuild_start, rebuild_last, - 0, NULL, true, true, true); - if (!ret) - svm->flush_tlb(svm); - - return ret; + return 0; } static void @@ -706,10 +701,11 @@ amdgpu_svm_range_process_notifier_ranges(struct amdgpu_svm *svm, if (clear_pte) { amdgpu_svm_range_gpu_unmap_in_notifier(svm, range, mmu_range); - range_invalidate_gpu_mapping(range); } drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx); + range_invalidate_gpu_mapping(range); + if (is_unmap) drm_gpusvm_range_set_unmapped(range, mmu_range); @@ -758,6 +754,7 @@ int amdgpu_svm_range_apply_attr_change(struct amdgpu_svm *svm, { lockdep_assert_held_write(&svm->svm_lock); + enum amdgpu_svm_migrate_mode migrate_mode = AMDGPU_SVM_MIGRATE_PREFERRED; bool old_access, new_access; bool update_mapping = false; @@ -789,15 +786,26 @@ int amdgpu_svm_range_apply_attr_change(struct amdgpu_svm *svm, update_mapping = true; if (trigger & AMDGPU_SVM_ATTR_TRIGGER_LOCATION_CHANGE) { - /* TODO: add migration */ + if (new_access) { + if (prefetch_loc_is_vram(new_attrs)) { + migrate_mode = AMDGPU_SVM_MIGRATE_TO_VRAM; + update_mapping = true; + } else if (new_attrs->prefetch_loc == + AMDGPU_SVM_LOCATION_SYSMEM) { + migrate_mode = AMDGPU_SVM_MIGRATE_TO_SYSMEM; + update_mapping = true; + } else { + AMDGPU_SVM_TRACE("no migration [0x%lx-0x%lx]\n", + start, last); + } + } } if (!update_mapping) return 0; - AMDGPU_SVM_TRACE("mapping update: remap interval [0x%lx-0x%lx]-0x%lx\n", - start, last, last - start + 1); - return amdgpu_svm_range_map_interval(svm, start, last, new_attrs); + return amdgpu_svm_range_map_interval(svm, start, last, new_attrs, + migrate_mode); } static bool @@ -1004,7 +1012,8 @@ static void amdgpu_svm_range_restore_worker(struct work_struct *w) down_write(&svm->svm_lock); ret = amdgpu_svm_range_map_attr_ranges(svm, op_ctx.start, - op_ctx.last); + op_ctx.last, + AMDGPU_SVM_MIGRATE_NONE); up_write(&svm->svm_lock); if (ret) { @@ -1042,7 +1051,6 @@ static void amdgpu_svm_range_restore_worker(struct work_struct *w) drm_gpusvm_notifier_unlock(&svm->gpusvm); svm->end_restore(svm); return; - } drm_gpusvm_notifier_unlock(&svm->gpusvm); } @@ -1099,6 +1107,15 @@ void amdgpu_svm_range_invalidate(struct amdgpu_svm *svm, if (atomic_read(&svm->exiting)) return; + /* + * Skip migration events that we initiated ourselves via + * drm_pagemap_populate_mm() (RAM→VRAM prefetch). + */ + if (mmu_range->event == MMU_NOTIFY_MIGRATE && + mmu_range->owner == AMDGPU_SVM_PGMAP_OWNER(svm->adev) && + atomic_read(&svm->in_populate)) + return; + if (!drm_gpusvm_range_find(notifier, mmu_range->start, mmu_range->end)) return; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h index 18bf3dad13fd..0065ae50c700 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h @@ -30,6 +30,8 @@ #include <linux/list.h> #include <linux/types.h> +#include "amdgpu_svm_range_migrate.h" + struct amdgpu_svm; struct amdgpu_svm_attrs; struct drm_gpusvm_notifier; @@ -62,7 +64,8 @@ void amdgpu_svm_range_flush(struct amdgpu_svm *svm); void amdgpu_svm_range_sync_work(struct amdgpu_svm *svm); int amdgpu_svm_range_map_attr_ranges(struct amdgpu_svm *svm, unsigned long start_page, - unsigned long last_page); + unsigned long last_page, + enum amdgpu_svm_migrate_mode migrate_mode); int amdgpu_svm_range_apply_attr_change( struct amdgpu_svm *svm, unsigned long start, unsigned long last, uint32_t trigger, const struct amdgpu_svm_attrs *prev_attrs, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.c new file mode 100644 index 000000000000..e7a2a11858f6 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.c @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include <drm/drm_pagemap.h> + +#include "amdgpu_svm.h" +#include "amdgpu_svm_attr.h" +#include "amdgpu_svm_range.h" +#include "amdgpu_svm_range_migrate.h" +#include "amdgpu.h" + +static bool +range_in_vram(struct drm_gpusvm_range *range) +{ + struct drm_gpusvm_pages_flags flags = { + /* Pairs with WRITE_ONCE in drm_gpusvm_get_pages() */ + .__flags = READ_ONCE(range->pages.flags.__flags), + }; + + return flags.has_devmem_pages; +} + +static bool +preferred_loc_is_vram(const struct amdgpu_svm_attrs *attrs) +{ + return attrs->preferred_loc != AMDGPU_SVM_LOCATION_SYSMEM && + attrs->preferred_loc != AMDGPU_SVM_LOCATION_UNDEFINED; +} + +static struct drm_pagemap * +amdgpu_svm_get_dpagemap(struct amdgpu_svm *svm) +{ + struct amdgpu_pagemap *apagemap = &svm->adev->kfd.apagemap; + + if (!apagemap->initialized) + return NULL; + + return &apagemap->dpagemap; +} + +/** + * range_needs_migrate_to_vram - Per-range VRAM migration check + * @svm: Pointer to the AMDGPU SVM structure + * @range: Pointer to the GPU SVM range + * @attrs: Current SVM attributes for this address range + * @mode: Migration mode (POLICY, TO_VRAM, or NONE) + * + * Three-level decision: + * 1. Capability: range->migrate_devmem (hardware, set at creation, stable) + * 2. Command: mode==TO_VRAM (one-shot prefetch_loc override) + * 3. Policy: preferred_loc (persistent user intent, when mode==POLICY) + * + * Return: true if range should be migrated to VRAM + */ +static bool +range_needs_migrate_to_vram(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct amdgpu_svm_attrs *attrs, + enum amdgpu_svm_migrate_mode mode) +{ + /* Capability: hardware must support VRAM for this range */ + if (!range->pages.flags.migrate_devmem) + return false; + + /* Suppress: post-evict remap or evict-to-SYSMEM — no VRAM migration */ + if (mode == AMDGPU_SVM_MIGRATE_NONE || + mode == AMDGPU_SVM_MIGRATE_TO_SYSMEM) + return false; + + /* Already backed by VRAM — no migration needed */ + if (range_in_vram(range)) + return false; + + /* Command: prefetch overrides policy — migrate unconditionally */ + if (mode == AMDGPU_SVM_MIGRATE_TO_VRAM) + return true; + + /* Policy: only migrate if preferred_loc targets VRAM */ + return preferred_loc_is_vram(attrs); +} + +/** + * amdgpu_svm_range_migrate_to_vram - Migrate a single range's pages to VRAM + * @svm: Pointer to the AMDGPU SVM structure + * @range: Pointer to the GPU SVM range to migrate + * + * The in_populate flag is set around populate_mm to tell the mmu_notifier + * handler to skip the self-migration event. After successful migration, + * the range's cached DMA addresses and gpu_mapped state are explicitly + * invalidated so that the subsequent get_pages() call will re-fault with + * fresh VRAM DMA addresses. + * + * Handles one retry-able error case: + * - -EBUSY: the range already has device-private pages from a different + * allocation. Evict those pages to RAM first and retry. + * + * Return: 0 on success, negative errno on failure + */ +static int +amdgpu_svm_range_migrate_to_vram(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range) +{ + struct drm_pagemap *dpagemap = amdgpu_svm_get_dpagemap(svm); + struct drm_gpusvm_ctx ctx = { .in_notifier = false }; + unsigned long start = drm_gpusvm_range_start(range); + unsigned long end = drm_gpusvm_range_end(range); + int ret, retries = 3; + + if (!dpagemap) + return -ENODEV; + + do { + atomic_set(&svm->in_populate, 1); + ret = drm_pagemap_populate_mm(dpagemap, + start, end, svm->gpusvm.mm, 0); + atomic_set(&svm->in_populate, 0); + + if (ret == -EBUSY && retries) { + AMDGPU_SVM_TRACE("migrate_to_vram -EBUSY, evicting [0x%lx-0x%lx]\n", + start, end); + drm_gpusvm_range_evict(&svm->gpusvm, range); + } else if (ret) { + break; + } + } while (ret && retries--); + + if (ret) { + AMDGPU_SVM_TRACE("migrate_to_vram failed: ret=%d [0x%lx-0x%lx]\n", + ret, start, end); + return ret; + } + + drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx); + WRITE_ONCE(to_amdgpu_svm_range(range)->gpu_mapped, false); + + return 0; +} + +bool +amdgpu_pagemap_capable(struct amdgpu_svm *svm) +{ + if (svm->adev->gmc.is_app_apu) + return false; + + if (!amdgpu_svm_get_dpagemap(svm)) + return false; + + return true; +} + +/** + * amdgpu_svm_range_migrate_range - Per-range migration for the map path + * @svm: Pointer to the AMDGPU SVM structure + * @range: The GPU SVM range to migrate + * @attrs: Current SVM attributes + * @migrate_mode: Migration intent + * @devmem_possible: Whether hardware supports device memory for this range + */ +int +amdgpu_svm_range_migrate_range(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct amdgpu_svm_attrs *attrs, + enum amdgpu_svm_migrate_mode migrate_mode) +{ + /* VRAM migration (TO_VRAM or POLICY with preferred_loc=VRAM) */ + if (range_needs_migrate_to_vram(svm, range, attrs, migrate_mode)) + return amdgpu_svm_range_migrate_to_vram(svm, range); + + /* VRAM eviction (prefetch-to-SYSMEM) */ + if (migrate_mode == AMDGPU_SVM_MIGRATE_TO_SYSMEM && + range_in_vram(range)) + return drm_gpusvm_range_evict(&svm->gpusvm, range); + + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.h new file mode 100644 index 000000000000..df14b0bc64da --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range_migrate.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_SVM_RANGE_MIGRATE_H__ +#define __AMDGPU_SVM_RANGE_MIGRATE_H__ + +struct amdgpu_svm; +struct amdgpu_svm_attrs; +struct drm_gpusvm_range; + +/** + * enum amdgpu_svm_migrate_mode - Migration intent for map paths + * @AMDGPU_SVM_MIGRATE_PREFERRED: Follow preferred_loc to decide migration + * @AMDGPU_SVM_MIGRATE_TO_VRAM: Force VRAM migration (prefetch-to-VRAM, one-shot) + * @AMDGPU_SVM_MIGRATE_TO_SYSMEM: Evict VRAM pages to system memory, then remap + * @AMDGPU_SVM_MIGRATE_NONE: Suppress all migration (post-evict remap — + * just map existing SYSMEM pages to GPU) + */ +enum amdgpu_svm_migrate_mode { + AMDGPU_SVM_MIGRATE_PREFERRED, + AMDGPU_SVM_MIGRATE_TO_VRAM, + AMDGPU_SVM_MIGRATE_TO_SYSMEM, + AMDGPU_SVM_MIGRATE_NONE, +}; + +bool amdgpu_pagemap_capable(struct amdgpu_svm *svm); +int amdgpu_svm_range_migrate_range(struct amdgpu_svm *svm, + struct drm_gpusvm_range *range, + const struct amdgpu_svm_attrs *attrs, + enum amdgpu_svm_migrate_mode migrate_mode); + +#endif /* __AMDGPU_SVM_RANGE_MIGRATE_H__ */ -- 2.34.1
