From: Honglei Huang <[email protected]> Make the VM eviction lock read/write to match the drm_gpusvm notifier lock. This allows VM updates, MMU notifier invalidation and eviction to share the same rwsem.
Replace the VM eviction mutex with a selectable rwsem. Updates made with the root BO reserved take the read side. Updates without the root BO reservation take the write side, as does eviction. A VM using drm_gpusvm can use the notifier lock as its eviction lock. Suggested-by: Christian König <[email protected]> Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 9 +++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 8 ++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h | 12 ++++++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index e40bce2912..a2cc6e2017 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -617,7 +617,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, * As soon as all page tables are in place we can start updating them * again. */ - scoped_guard(mutex, &vm->eviction_lock) + scoped_guard(rwsem_write, vm->eviction_lock) vm->evicting = false; list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted, @@ -678,7 +678,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm) amdgpu_vm_assert_locked(vm); - scoped_guard(mutex, &vm->eviction_lock) + scoped_guard(rwsem_read, vm->eviction_lock) ret = !vm->evicting; ret &= list_empty(&vm->kernel.evicted); @@ -2351,7 +2351,7 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo) /* Try to block ongoing updates */ vm = bo_base->vm; - scoped_cond_guard(mutex_try, return false, &vm->eviction_lock) { + scoped_cond_guard(rwsem_write_try, return false, vm->eviction_lock) { /* Don't evict VM page tables while they are updated */ if (!dma_fence_is_signaled(vm->last_unlocked)) @@ -2707,7 +2707,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, vm->last_tlb_flush = dma_fence_get_stub(); vm->generation = amdgpu_vm_generation(adev, NULL); - mutex_init(&vm->eviction_lock); + init_rwsem(&vm->default_eviction_lock); + vm->eviction_lock = &vm->default_eviction_lock; vm->evicting = false; vm->tlb_fence_context = dma_fence_context_alloc(1); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index f5cf2dd374..eac2632db0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -27,6 +27,7 @@ #include <linux/idr.h> #include <linux/kfifo.h> #include <linux/rbtree.h> +#include <linux/rwsem.h> #include <drm/gpu_scheduler.h> #include <drm/drm_file.h> #include <drm/ttm/ttm_bo.h> @@ -283,10 +284,9 @@ struct amdgpu_vm { /* tree of virtual addresses mapped */ struct rb_root_cached va; - /* Lock to prevent eviction while we are updating page tables - * use vm_eviction_lock/unlock(vm) - */ - struct mutex eviction_lock; + /* SVM can replace this pointer with its notifier lock. */ + struct rw_semaphore default_eviction_lock; + struct rw_semaphore *eviction_lock; bool evicting; /* Memory statistics for this vm, protected by stats_lock */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h index 3d4a80fd76..2bd069c56c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h @@ -157,7 +157,12 @@ void amdgpu_vm_pt_free_dummies(struct amdgpu_device *adev); */ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p) { - mutex_lock(&p->vm->eviction_lock); + /* Use the write lock to serialize updates when the root BO is unlocked. */ + if (p->unlocked) + down_write(p->vm->eviction_lock); + else + down_read(p->vm->eviction_lock); + p->saved_flags = memalloc_noreclaim_save(); if (p->vm->evicting) return -EBUSY; @@ -175,7 +180,10 @@ static inline int amdgpu_vm_begin_critical(struct amdgpu_vm_update_params *p) static inline void amdgpu_vm_end_critical(struct amdgpu_vm_update_params *p) { memalloc_noreclaim_restore(p->saved_flags); - mutex_unlock(&p->vm->eviction_lock); + if (p->unlocked) + up_write(p->vm->eviction_lock); + else + up_read(p->vm->eviction_lock); } #endif -- 2.34.1
