From: Junhua Shen <[email protected]> Implement the TTM eviction path for SVM (Shared Virtual Memory) BOs, enabling VRAM overcommit scenarios where device-private pages must be migrated back to system memory when VRAM is full.
- amdgpu_ttm.c: Add SVM BO detection in amdgpu_evict_flags() to evict directly to SYSTEM domain. Add SVM BO handling in amdgpu_bo_move() to call amdgpu_svm_bo_evict() for VRAM->SYSTEM transitions, which synchronously migrates device-private pages back to RAM via SDMA. - amdgpu_migrate.c: Add amdgpu_svm_bo_evict() and amdgpu_svm_bo_is_svm() helpers. - amdgpu_object.c: Register SVM BO destroy callback in amdgpu_bo_is_amdgpu_bo() so TTM eviction can identify and handle SVM BOs correctly. Signed-off-by: Junhua Shen <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.c | 46 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.h | 12 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 20 +++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.c index 20daf2922e0f6..cd7483ad3cc3d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.c @@ -64,6 +64,7 @@ #include <drm/drm_pagemap.h> #include <linux/memremap.h> #include <linux/migrate.h> +#include <linux/sched/mm.h> #include "amdgpu_migrate.h" #include "amdgpu.h" @@ -597,6 +598,51 @@ static const struct drm_pagemap_devmem_ops amdgpu_pagemap_ops = { .copy_to_ram = amdgpu_svm_copy_to_ram, }; +/** + * amdgpu_svm_bo_evict - Evict SVM BO by migrating device pages back to RAM + * @bo: The amdgpu_bo to evict (must be an amdgpu_bo_svm) + * + * Called from amdgpu_bo_move() when TTM needs to evict a SVM BO from VRAM. + * Uses drm_pagemap_evict_to_ram() to synchronously migrate all device-private + * pages back to system memory via SDMA. + * + * Return: 0 on success, negative error code on failure + */ +int amdgpu_svm_bo_evict(struct amdgpu_bo *bo) +{ + struct amdgpu_bo_svm *sbo = to_amdgpu_bo_svm(bo); + struct mm_struct *mm = sbo->devmem.mm; + int r; + + /* + * If the owning process is exiting, there is nothing to migrate; return + * -EBUSY so the TTM eviction LRU walk skips this BO instead of aborting + * the whole walk (which would fail the allocation that triggered the + * eviction). Hold an mm reference across the call so the framework's own + * dead-mm path stays unreachable and any -EFAULT it returns is + * unambiguously a real error. + */ + if (!mmget_not_zero(mm)) + return -EBUSY; + + r = drm_pagemap_evict_to_ram(&sbo->devmem); + + mmput_async(mm); + + return r; +} + +/** + * amdgpu_svm_bo_is_svm - Check if a TTM BO is an SVM BO + * @tbo: TTM buffer object + * + * Return: true if this is an amdgpu_bo_svm (identified by destroy callback) + */ +bool amdgpu_svm_bo_is_svm(struct ttm_buffer_object *tbo) +{ + return tbo->destroy == &amdgpu_bo_svm_destroy; +} + /* drm_pagemap_ops — top-level migration entry points */ /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.h index 56c66851f0a47..e9159e8167dd7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_migrate.h @@ -30,6 +30,8 @@ #include <linux/workqueue.h> struct amdgpu_device; +struct amdgpu_bo; +struct ttm_buffer_object; #define AMDGPU_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER #define AMDGPU_INTERCONNECT_P2P (AMDGPU_INTERCONNECT_VRAM + 1) @@ -75,12 +77,22 @@ struct amdgpu_pagemap { #if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM) int amdgpu_svm_migration_init(struct amdgpu_device *adev); +int amdgpu_svm_bo_evict(struct amdgpu_bo *bo); +bool amdgpu_svm_bo_is_svm(struct ttm_buffer_object *tbo); #else static inline int amdgpu_svm_migration_init(struct amdgpu_device *adev) { return 0; } +static inline int amdgpu_svm_bo_evict(struct amdgpu_bo *bo) +{ + return -ENODEV; +} +static inline bool amdgpu_svm_bo_is_svm(struct ttm_buffer_object *tbo) +{ + return false; +} #endif /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index fc6d3fa62d794..2615946165bb1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -44,6 +44,7 @@ #include "amdgpu_vm.h" #include "amdgpu_dma_buf.h" #include "kfd_svm.h" +#include "amdgpu_migrate.h" /** * DOC: amdgpu_object @@ -95,7 +96,8 @@ bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) { if (bo->destroy == &amdgpu_bo_destroy || bo->destroy == &amdgpu_bo_user_destroy || - bo->destroy == &svm_range_bo_destroy) + bo->destroy == &svm_range_bo_destroy || + amdgpu_svm_bo_is_svm(bo)) return true; return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 3d620ec2937f4..6f1b109b3fd83 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -60,6 +60,7 @@ #include "amdgpu_hmm.h" #include "amdgpu_atomfirmware.h" #include "amdgpu_res_cursor.h" +#include "amdgpu_migrate.h" #include "bif/bif_4_1_d.h" #include "kfd_svm.h" @@ -123,6 +124,15 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo, return; } + /* SVM BOs must evict directly to system memory for drm_pagemap + * migration back to RAM in amdgpu_bo_move(). + */ + if (amdgpu_svm_bo_is_svm(bo)) { + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); + *placement = abo->placement; + return; + } + switch (bo->resource->mem_type) { case AMDGPU_PL_GDS: case AMDGPU_PL_GWS: @@ -610,6 +620,16 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; } + /* SVM BO eviction: migrate device-private pages back to RAM */ + if (amdgpu_svm_bo_is_svm(bo) && + old_mem->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM) { + r = amdgpu_svm_bo_evict(abo); + if (!r) + ttm_bo_move_null(bo, new_mem); + return r; + } + if (adev->mman.buffer_funcs_enabled && ((old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_VRAM) || -- 2.53.0
