This will allow the use of all of them for clear/fill buffer operations. Since drm_sched_entity_init requires a scheduler array, we store schedulers rather than rings. For the few places that need access to a ring, we can get it from the sched using container_of.
Since the code is the same for all sdma versions, add a new helper amdgpu_sdma_set_buffer_funcs_scheds to set buffer_funcs_scheds based on the number of sdma instances. Note: the new sched array is identical to the amdgpu_vm_manager one. These 2 could be merged. Signed-off-by: Pierre-Eric Pelloux-Prayer <[email protected]> Acked-by: Felix Kuehling <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 4 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 32 ++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 +- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 3 +- drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 3 +- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 3 +- drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c | 6 +--- drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c | 6 +--- drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c | 5 +--- drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c | 5 +--- drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 3 +- drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 3 +- drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c | 3 +- drivers/gpu/drm/amd/amdgpu/si_dma.c | 3 +- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 3 +- 17 files changed, 47 insertions(+), 42 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index af4042387f3b..5275311eb09b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1538,6 +1538,8 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring); ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset); void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev, const struct amdgpu_vm_pte_funcs *vm_pte_funcs); +void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev, + const struct amdgpu_buffer_funcs *buffer_funcs); /* atpx handler */ #if defined(CONFIG_VGA_SWITCHEROO) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 5e73b9d67325..b5c48ff3d67e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4390,7 +4390,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->num_rings = 0; RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); adev->mman.buffer_funcs = NULL; - adev->mman.buffer_funcs_ring = NULL; + adev->mman.num_buffer_funcs_scheds = 0; adev->vm_manager.vm_pte_funcs = NULL; adev->vm_manager.vm_pte_num_scheds = 0; adev->gmc.gmc_funcs = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index e87cb8ccfcd9..f52c764d4c10 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -707,12 +707,14 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, uint32_t vmhub, uint32_t flush_type) { - struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + struct amdgpu_ring *ring; struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; struct dma_fence *fence; struct amdgpu_job *job; int r; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); + if (!hub->sdma_invalidation_workaround || vmid || !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || !ring->sched.ready) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 91fcf4f08181..6df3a4659172 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -168,7 +168,7 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entit { struct amdgpu_ring *ring; - ring = adev->mman.buffer_funcs_ring; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); amdgpu_ring_pad_ib(ring, &job->ibs[0]); WARN_ON(job->ibs[0].length_dw > num_dw); @@ -2359,18 +2359,17 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) return; if (enable) { - struct amdgpu_ring *ring; struct drm_gpu_scheduler *sched; - if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) { + if (!adev->mman.num_buffer_funcs_scheds || + !adev->mman.buffer_funcs_scheds[0]->ready) { dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use"); return; } num_clear_entities = 1; num_move_entities = 1; - ring = adev->mman.buffer_funcs_ring; - sched = &ring->sched; + sched = adev->mman.buffer_funcs_scheds[0]; r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr, &adev->mman.default_entity, DRM_SCHED_PRIORITY_KERNEL, @@ -2507,7 +2506,7 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev, unsigned int i; int r; - ring = adev->mman.buffer_funcs_ring; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); if (!ring->sched.ready) { dev_err(adev->dev, @@ -2740,6 +2739,27 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type) return ttm_resource_manager_evict_all(&adev->mman.bdev, man); } +void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev, + const struct amdgpu_buffer_funcs *buffer_funcs) +{ + struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; + struct drm_gpu_scheduler *sched; + int i; + + adev->mman.buffer_funcs = buffer_funcs; + + for (i = 0; i < adev->sdma.num_instances; i++) { + if (adev->sdma.has_page_queue) + sched = &adev->sdma.instance[i].page.sched; + else + sched = &adev->sdma.instance[i].ring.sched; + adev->mman.buffer_funcs_scheds[i] = sched; + } + + adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ? + 1 : adev->sdma.num_instances; +} + #if defined(CONFIG_DEBUG_FS) static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index 3b1973611446..a6249252948b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -67,7 +67,8 @@ struct amdgpu_mman { /* buffer handling */ const struct amdgpu_buffer_funcs *buffer_funcs; - struct amdgpu_ring *buffer_funcs_ring; + struct drm_gpu_scheduler *buffer_funcs_scheds[AMDGPU_MAX_RINGS]; + u32 num_buffer_funcs_scheds; bool buffer_funcs_enabled; /* @default_entity: for workarounds, has no gart windows */ diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 22780c09177d..26276dcfd458 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -1340,8 +1340,7 @@ static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = { static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &cik_sdma_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &cik_sdma_buffer_funcs); } const struct amdgpu_ip_block_version cik_sdma_ip_block = diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 0090ace49024..c6a059ca59e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -1235,8 +1235,7 @@ static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = { static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v2_4_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v2_4_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 2526d393162a..cb516a25210d 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -1677,8 +1677,7 @@ static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = { static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v3_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v3_0_ip_block = diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index a35d9951e22a..f234ee54f39e 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -2615,11 +2615,7 @@ static const struct amdgpu_buffer_funcs sdma_v4_0_buffer_funcs = { static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs; - if (adev->sdma.has_page_queue) - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page; - else - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_0_buffer_funcs); } static void sdma_v4_0_get_ras_error_count(uint32_t value, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index 7f77367848d4..cd7627b03066 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -2316,11 +2316,7 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = { static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs; - if (adev->sdma.has_page_queue) - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page; - else - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_2_buffer_funcs); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index d72bd3adfccf..5da18f845014 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -2061,10 +2061,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = { static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev) { - if (adev->mman.buffer_funcs == NULL) { - adev->mman.buffer_funcs = &sdma_v5_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; - } + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v5_0_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 5aa500fe554b..4133163fa24c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -2072,10 +2072,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = { static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev) { - if (adev->mman.buffer_funcs == NULL) { - adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; - } + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_2_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v5_2_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 45d13ac09f9b..1b9c1e659095 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -1904,8 +1904,7 @@ static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = { static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v6_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v6_0_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index f938be0524cd..0896f0dc6030 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -1845,8 +1845,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = { static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v7_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v7_0_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c index 3de76afe3e45..ac63dc26ed53 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c @@ -1755,8 +1755,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = { static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v7_1_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_1_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v7_1_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c index 3e58feb2d5e4..155067c20a0e 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dma.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c @@ -833,8 +833,7 @@ static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = { static void si_dma_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &si_dma_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs); } const struct amdgpu_ip_block_version si_dma_ip_block = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 7b089d22c367..f3e6ec68b167 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -129,13 +129,14 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys, struct dma_fence **mfence) { const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE; - struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + struct amdgpu_ring *ring; struct amdgpu_ttm_buffer_entity *entity; u64 gart_s, gart_d; struct dma_fence *next; u64 size; int r; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); entity = &adev->mman.move_entities[0]; mutex_lock(&entity->lock); -- 2.43.0
