On 11/9/26 17:26, Zhigang Luo wrote:
> On the A+A VF the hypervisor reports each VF's VRAM address in the
> ACPI SRAT table, and that address is stored in aper_base. The XGMI
> node segment does not describe the VF, so physical_node_id *
> node_segment_size is not a valid VRAM starting address.
>
> gmc_v12_0_vram_gtt_location() and amdgpu_virt_resume() still added
> that offset on top of aper_base, which moved the SPM PDB0 system PTEs
> off the VF's HBM. Leave vram_base_offset at aper_base on both paths.
> Size PDB0 from the VF's local VRAM and place GART after that window,
> aligned to 8GB. That alignment is one PDB0 entry when
> vmid0_page_table_block_size is 12, and it applies to every PDB0
> placement.
>
> v2:
> - Explain that the SRAT address is already the VF VRAM base and must
> not be adjusted by the XGMI node segment.
>
> Signed-off-by: Zhigang Luo <[email protected]>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++---
> drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 22 ++++++++++++++++------
> drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 6 ++++--
> 3 files changed, 29 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 9326f811c500..7a664b7b1ba3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -4677,9 +4677,15 @@ static inline int amdgpu_virt_resume(struct
> amdgpu_device *adev)
> dev_info(adev->dev, "xgmi node, old id %d, new id %d\n",
> prev_physical_node_id, adev->gmc.xgmi.physical_node_id);
>
> - adev->vm_manager.vram_base_offset =
> adev->gfxhub.funcs->get_mc_fb_offset(adev);
> - adev->vm_manager.vram_base_offset +=
> - adev->gmc.xgmi.physical_node_id *
> adev->gmc.xgmi.node_segment_size;
> + if (amdgpu_virt_vram_is_spm(adev)) {
> + adev->vm_manager.vram_base_offset = adev->gmc.aper_base;
> + } else {
> + adev->vm_manager.vram_base_offset =
> + adev->gfxhub.funcs->get_mc_fb_offset(adev);
> + adev->vm_manager.vram_base_offset +=
> + adev->gmc.xgmi.physical_node_id *
> + adev->gmc.xgmi.node_segment_size;
This is a massive NAK. That needs to be handled in get_mc_fb_offset().
See how aper_base is initialized on APUS as well:
#ifdef CONFIG_X86_64
if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev);
adev->gmc.aper_size = adev->gmc.real_vram_size;
}
#endif
> + }
>
> return 0;
> }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index f76da74abfec..96075db3b9ac 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -42,6 +42,8 @@
> #include <drm/ttm/ttm_tt.h>
>
> static const u64 four_gb = 0x100000000ULL;
> +/* PDB0 block size 12 covers one 8GB entry. */
> +static const u64 eight_gb = 0x200000000ULL;
>
> bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev)
> {
> @@ -60,7 +62,8 @@ int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev)
> {
> int r;
> struct amdgpu_bo_param bp;
> - u64 vram_size = adev->gmc.xgmi.node_segment_size *
> adev->gmc.xgmi.num_physical_nodes;
> + u64 vram_size = amdgpu_virt_vram_is_spm(adev) ? adev->gmc.mc_vram_size :
> + adev->gmc.xgmi.node_segment_size *
> adev->gmc.xgmi.num_physical_nodes;
Same here. I need to double check but we probably need a new callback for this.
Regards,
Christian.
> uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21;
> uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >>
> pde0_page_shift;
>
> @@ -261,16 +264,21 @@ void amdgpu_gmc_sysvm_location(struct amdgpu_device
> *adev, struct amdgpu_gmc *mc
> u64 hive_vram_end = mc->xgmi.node_segment_size *
> mc->xgmi.num_physical_nodes - 1;
> mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id;
> mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1;
> - /* node_segment_size may not 4GB aligned on SRIOV, align up is needed.
> */
> - mc->gart_start = ALIGN(hive_vram_end + 1, four_gb);
> + /* Round up to a PDB0 entry so GART starts on the page-table PDE. */
> + mc->gart_start = ALIGN(hive_vram_end + 1, eight_gb);
> mc->gart_end = mc->gart_start + mc->gart_size - 1;
> - if (amdgpu_virt_xgmi_migrate_enabled(adev)) {
> + if (amdgpu_virt_xgmi_migrate_enabled(adev) ||
> + amdgpu_virt_vram_is_spm(adev)) {
> /* set mc->vram_start to 0 to switch the returned GPU address of
> * amdgpu_bo_create_reserved() from FB aperture to GART
> aperture.
> */
> mc->vram_start = 0;
> mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
> mc->visible_vram_size = min(mc->visible_vram_size,
> mc->real_vram_size);
> + if (amdgpu_virt_vram_is_spm(adev)) {
> + mc->gart_start = ALIGN(mc->vram_end + 1, eight_gb);
> + mc->gart_end = mc->gart_start + mc->gart_size - 1;
> + }
> } else {
> mc->fb_start = hive_vram_start;
> mc->fb_end = hive_vram_end;
> @@ -1269,7 +1277,8 @@ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
> uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore
> NC/RW?
> /* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M
> */
> - u64 vram_size = adev->gmc.xgmi.node_segment_size *
> adev->gmc.xgmi.num_physical_nodes;
> + u64 vram_size = amdgpu_virt_vram_is_spm(adev) ? adev->gmc.mc_vram_size :
> + adev->gmc.xgmi.node_segment_size *
> adev->gmc.xgmi.num_physical_nodes;
> u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21;
> u64 vram_addr, vram_end;
> u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo);
> @@ -1287,7 +1296,8 @@ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
> flags |= AMDGPU_PDE_PTE_FLAG(adev);
>
> vram_addr = adev->vm_manager.vram_base_offset;
> - if (!amdgpu_virt_xgmi_migrate_enabled(adev))
> + if (!amdgpu_virt_xgmi_migrate_enabled(adev) &&
> + !amdgpu_virt_vram_is_spm(adev))
> vram_addr -= adev->gmc.xgmi.physical_node_id *
> adev->gmc.xgmi.node_segment_size;
> vram_end = vram_addr + vram_size;
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> index cd276a138e10..e37c84f10709 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> @@ -734,8 +734,10 @@ static void gmc_v12_0_vram_gtt_location(struct
> amdgpu_device *adev,
> adev->vm_manager.vram_base_offset =
> adev->mmhub.funcs->get_mc_fb_offset(adev);
> }
>
> - adev->vm_manager.vram_base_offset +=
> - adev->gmc.xgmi.physical_node_id *
> adev->gmc.xgmi.node_segment_size;
> + if (!amdgpu_virt_vram_is_spm(adev))
> + adev->vm_manager.vram_base_offset +=
> + adev->gmc.xgmi.physical_node_id *
> + adev->gmc.xgmi.node_segment_size;
> }
>
> /**
