Re: [PATCH] amdgpu/amdkfd: peerdirect support address lookup

[Chen, Xiaogang]

On 9/4/2025 9:45 PM, David Yat Sin wrote:
Add look-up for the BO by address. This is required for memory
allocations where the virtual address are reserved outside of
KFD and then later mapped to GPU memory.

Signed-off-by: David Yat Sin <david.yat...@amd.com>
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 41 ++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h     |  5 ++
  drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c | 61 +++++++++++++--------
  3 files changed, 83 insertions(+), 24 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 81afeda39667..37a2beece6ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -2667,8 +2667,49 @@ int amdgpu_mmap(struct file *filp, struct vm_area_struct 
*vma)
          vma->vm_ops = &amdgpu_ttm_vm_ops;
          return 0;
  }
+
+struct amdgpu_bo *amdgpu_bo_from_address(struct mm_struct *mm,
+                                                       unsigned long addr,
+                                                       unsigned long *offset,
+                                                       struct amdgpu_device 
**adev)
+{
+       struct ttm_buffer_object *bo;
+       struct vm_area_struct *vma;
+       struct amdgpu_bo *abo = NULL;
+
+       mmap_read_lock(mm);
+       vma = find_vma(mm, addr);
find_vma finds mapping from cpu vm, not gpu vm that the commit message 
indicates. The addr needs be mmaped by drm into process cpu vm, then 
find_vma can find it. Is the addr here has been mapped into cpu vm?

+       if (!vma)
+               goto exit;
+
+       /* Confirm that this VMA belongs to amdgpu */
+       if (vma->vm_ops != &amdgpu_ttm_vm_ops)
+               goto exit;
+
Is this patch supposed to be sent to this mailing list? The code here is 
for RDMA support from kfd. I think it has not been upstream.

Regards

Xiaogang


+       abo = ttm_to_amdgpu_bo(vma->vm_private_data);
+
+       if (offset)
+               *offset = addr - vma->vm_start;
+
+       if (adev)
+               *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+exit:
+       mmap_read_unlock(mm);
+       return abo;
+}
+#else
+struct amdgpu_bo *amdgpu_bo_from_address(struct mm_struct *mm,
+                                                       unsigned long addr,
+                                                       unsigned long *offset,
+                                                       struct amdgpu_device 
**adev)
+{
+       pr_err("BO from address look-up not supported\n");
+       return NULL;
+}
  #endif /* HAVE_STRUCT_DRM_DRV_GEM_OPEN_OBJECT_CALLBACK */
  
+
+
  int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
                       uint64_t dst_offset, uint32_t byte_count,
                       struct dma_resv *resv,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 747c9669069a..9fbff8dec861 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -175,6 +175,11 @@ int amdgpu_ttm_init(struct amdgpu_device *adev);
  void amdgpu_ttm_fini(struct amdgpu_device *adev);
  void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,
                                        bool enable);
+
+struct amdgpu_bo *amdgpu_bo_from_address(struct mm_struct *mm,
+                                       unsigned long addr, unsigned long 
*offset,
+                                       struct amdgpu_device **adev);
+
  int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
                       uint64_t dst_offset, uint32_t byte_count,
                       struct dma_resv *resv,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c 
b/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
index ed93247d83ca..3341f40f2a23 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
@@ -170,9 +170,13 @@ static int amd_acquire(unsigned long addr, size_t size,
                        void *peer_mem_private_data,
                        char *peer_mem_name, void **client_context)
  {
-       struct kfd_process *p;
-       struct kfd_bo *buf_obj;
        struct amd_mem_context *mem_context;
+       struct kfd_process *p;
+       struct kfd_bo *buf_obj = NULL;
+       struct kfd_node *dev = NULL;
+       struct amdgpu_bo *bo = NULL;
+       unsigned long offset = 0;
+       uint32_t flags = 0;
  
  	if (peer_mem_name == rdma_name) {
                p = peer_mem_private_data;
@@ -187,34 +191,44 @@ static int amd_acquire(unsigned long addr, size_t size,
        align_addr_size(&addr, &size);
  
  	mutex_lock(&p->mutex);
-       buf_obj = kfd_process_find_bo_from_interval(p, addr,
-                       addr + size - 1);
-       if (!buf_obj) {
-               pr_debug("Cannot find a kfd_bo for the range\n");
-               goto out_unlock;
+       buf_obj = kfd_process_find_bo_from_interval(p, addr, addr + size - 1);
+       if (buf_obj) {
+               offset = addr - buf_obj->it.start;
+               bo = amdgpu_amdkfd_gpuvm_get_bo_ref(buf_obj->mem, &flags);
+
+               dev = buf_obj->dev;
+       } else {
+               struct amdgpu_device *adev;
+               /* Memory was allocated via KFD without address, then mapped 
later */
+               bo = amdgpu_bo_from_address(p->mm, addr, &offset, &adev);
+               if (!bo)
+                       goto out_unlock;
+
+               flags = bo->kfd_bo->alloc_flags;
+               dev = adev->kfd.dev->nodes[bo->xcp_id];
+               drm_gem_object_get(&bo->tbo.base);
        }
  
-	/* Initialize context used for operation with given address */
        mem_context = kzalloc(sizeof(*mem_context), GFP_KERNEL);
-       if (!mem_context)
-               goto out_unlock;
+       if (unlikely(!mem_context)) {
+               if (buf_obj)
+                       amdgpu_amdkfd_gpuvm_put_bo_ref(bo);
+               else
+                       drm_gem_object_put(&bo->tbo.base);
  
-	mem_context->pid = p->lead_thread->pid;
-
-       pr_debug("addr: %#lx, size: %#lx, pid: %d\n",
-                addr, size, mem_context->pid);
-
-       mem_context->va     = addr;
-       mem_context->size   = size;
-       mem_context->offset = addr - buf_obj->it.start;
-
-       mem_context->bo = amdgpu_amdkfd_gpuvm_get_bo_ref(buf_obj->mem,
-                                                        &mem_context->flags);
-       mem_context->dev = buf_obj->dev;
+               goto out_unlock;
+       }
  
  	mutex_unlock(&p->mutex);
+       pr_debug("addr: %#lx, size: %#lx, pid: %d\n", addr, size, 
mem_context->pid);
  
-	pr_debug("Client context: 0x%p\n", mem_context);
+       mem_context->pid = p->lead_thread->pid;
+       mem_context->va = addr;
+       mem_context->size = size;
+       mem_context->dev = dev;
+       mem_context->offset = offset;
+       mem_context->bo = bo;
+       mem_context->flags = flags;
  
  	/* Return pointer to allocated context */
        *client_context = mem_context;
@@ -223,7 +237,6 @@ static int amd_acquire(unsigned long addr, size_t size,
         * by AMD GPU driver
         */
        return 1;
-
  out_unlock:
        mutex_unlock(&p->mutex);
        return 0;