From: Honglei Huang <[email protected]> Make the VM eviction lock read/write to match the drm_gpusvm notifier lock. This allows VM updates, MMU notifier invalidation and eviction to share the same rwsem.
Replace the VM eviction mutex with a selectable rwsem. Updates made with the root BO reserved take the read side. Updates without the root BO reservation take the write side, as does eviction. A VM using drm_gpusvm can use the notifier lock as its eviction lock. Suggested-by: Christian König <[email protected]> Suggested-by: Huang Rui <[email protected]> Signed-off-by: Yu Huang <[email protected]> Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 9 +++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 8 ++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h | 12 ++++++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 77f1794eb5011..e851912999a51 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -617,7 +617,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, * As soon as all page tables are in place we can start updating them * again. */ - scoped_guard(mutex, &vm->eviction_lock) + scoped_guard(rwsem_write, vm->eviction_lock) vm->evicting = false; list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted, @@ -678,7 +678,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm) amdgpu_vm_assert_locked(vm); - scoped_guard(mutex, &vm->eviction_lock) + scoped_guard(rwsem_read, vm->eviction_lock) ret = !vm->evicting; ret &= list_empty(&vm->kernel.evicted); @@ -2357,7 +2357,7 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo) /* Try to block ongoing updates */ vm = bo_base->vm; - scoped_cond_guard(mutex_try, return false, &vm->eviction_lock) { + scoped_cond_guard(rwsem_write_try, return false, vm->eviction_lock) { /* Don't evict VM page tables while they are updated */ if (!dma_fence_is_signaled(vm->last_unlocked)) @@ -2713,7 +2713,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, vm->last_tlb_flush = dma_fence_get_stub(); vm->generation = amdgpu_vm_generation(adev, NULL); - mutex_init(&vm->eviction_lock); + init_rwsem(&vm->default_eviction_lock); + vm->eviction_lock = &vm->default_eviction_lock; vm->evicting = false; vm->tlb_fence_context = dma_fence_context_alloc(1); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index b435e393be154..ffdfaadc90544 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -27,6 +27,7 @@ #include <linux/idr.h> #include <linux/kfifo.h> #include <linux/rbtree.h> +#include <linux/rwsem.h> #include <drm/gpu_scheduler.h> #include <drm/drm_file.h> #include <drm/ttm/ttm_bo.h> @@ -288,10 +289,9 @@ struct amdgpu_vm { /* tree of virtual addresses mapped */ struct rb_root_cached va; - /* Lock to prevent eviction while we are updating page tables - * use vm_eviction_lock/unlock(vm) - */ - struct mutex eviction_lock; + /* SVM can replace this pointer with its notifier lock. */ + struct rw_semaphore default_eviction_lock; + struct rw_semaphore *eviction_lock; bool evicting; /* Memory statistics for this vm, protected by stats_lock */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h index 8a0e5fcbd55ba..920d0b3935807 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h @@ -160,7 +160,12 @@ void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev); */ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p) { - mutex_lock(&p->vm->eviction_lock); + /* Use the write lock to serialize updates when the root BO is unlocked. */ + if (p->unlocked) + down_write(p->vm->eviction_lock); + else + down_read(p->vm->eviction_lock); + p->saved_flags = memalloc_noreclaim_save(); if (p->vm->evicting) return -EBUSY; @@ -178,7 +183,10 @@ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p) static inline void amdgpu_vm_end_critical(struct amdgpu_vm_update_params *p) { memalloc_noreclaim_restore(p->saved_flags); - mutex_unlock(&p->vm->eviction_lock); + if (p->unlocked) + up_write(p->vm->eviction_lock); + else + up_read(p->vm->eviction_lock); } #endif -- 2.53.0
