Hi Shashank,
Inline:
On 2023-03-30 07:09, Christian König wrote:
> Am 29.03.23 um 17:47 schrieb Shashank Sharma:
>> From: Shashank Sharma <contactshashanksha...@gmail.com>
>>
>> This patch:
>> - creates a new file for doorbell management.
>> - moves doorbell code from amdgpu_device.c to this file.
>>
>> Cc: Alex Deucher <alexander.deuc...@amd.com>
>> Cc: Christian Koenig <christian.koe...@amd.com>
>> Signed-off-by: Shashank Sharma <shashank.sha...@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 164 ---------------
>> drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h | 22 +++
>> .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c | 186 ++++++++++++++++++
>> 4 files changed, 209 insertions(+), 165 deletions(-)
>> create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>> index 798d0e9a60b7..204665f20319 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>> @@ -41,7 +41,7 @@ ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \
>> amdgpu-y := amdgpu_drv.o
>>
>> # add KMS driver
>> -amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>> +amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
>> amdgpu_atombios.o atombios_crtc.o amdgpu_connectors.o \
>> atom.o amdgpu_fence.o amdgpu_ttm.o amdgpu_object.o amdgpu_gart.o \
>> amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 57ee1c4a81e9..7f8fcac4f18b 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -579,94 +579,6 @@ void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
>> }
>> }
>>
>> -/**
>> - * amdgpu_mm_rdoorbell - read a doorbell dword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - *
>> - * Returns the value in the doorbell aperture at the
>> - * requested doorbell index (CIK).
>> - */
>> -u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
>> -{
>> - if (amdgpu_device_skip_hw_access(adev))
>> - return 0;
>> -
>> - if (index < adev->doorbell.num_kernel_doorbells) {
>> - return readl(adev->doorbell.ptr + index);
>> - } else {
>> - DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>> - return 0;
>> - }
>> -}
>> -
>> -/**
>> - * amdgpu_mm_wdoorbell - write a doorbell dword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - * @v: value to write
>> - *
>> - * Writes @v to the doorbell aperture at the
>> - * requested doorbell index (CIK).
>> - */
>> -void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
>> -{
>> - if (amdgpu_device_skip_hw_access(adev))
>> - return;
>> -
>> - if (index < adev->doorbell.num_kernel_doorbells) {
>> - writel(v, adev->doorbell.ptr + index);
>> - } else {
>> - DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>> - }
>> -}
>> -
>> -/**
>> - * amdgpu_mm_rdoorbell64 - read a doorbell Qword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - *
>> - * Returns the value in the doorbell aperture at the
>> - * requested doorbell index (VEGA10+).
>> - */
>> -u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
>> -{
>> - if (amdgpu_device_skip_hw_access(adev))
>> - return 0;
>> -
>> - if (index < adev->doorbell.num_kernel_doorbells) {
>> - return atomic64_read((atomic64_t *)(adev->doorbell.ptr +
>> index));
>> - } else {
>> - DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>> - return 0;
>> - }
>> -}
>> -
>> -/**
>> - * amdgpu_mm_wdoorbell64 - write a doorbell Qword
>> - *
>> - * @adev: amdgpu_device pointer
>> - * @index: doorbell index
>> - * @v: value to write
>> - *
>> - * Writes @v to the doorbell aperture at the
>> - * requested doorbell index (VEGA10+).
>> - */
>> -void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
>> -{
>> - if (amdgpu_device_skip_hw_access(adev))
>> - return;
>> -
>> - if (index < adev->doorbell.num_kernel_doorbells) {
>> - atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>> - } else {
>> - DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>> - }
>> -}
>> -
>> /**
>> * amdgpu_device_indirect_rreg - read an indirect register
>> *
>> @@ -1016,82 +928,6 @@ int amdgpu_device_pci_reset(struct amdgpu_device *adev)
>> return pci_reset_function(adev->pdev);
>> }
>>
>> -/*
>> - * GPU doorbell aperture helpers function.
>> - */
>> -/**
>> - * amdgpu_device_doorbell_init - Init doorbell driver information.
>> - *
>> - * @adev: amdgpu_device pointer
>> - *
>> - * Init doorbell driver information (CIK)
>> - * Returns 0 on success, error on failure.
>> - */
>> -static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
>> -{
>> -
>> - /* No doorbell on SI hardware generation */
>> - if (adev->asic_type < CHIP_BONAIRE) {
>> - adev->doorbell.base = 0;
>> - adev->doorbell.size = 0;
>> - adev->doorbell.num_kernel_doorbells = 0;
>> - adev->doorbell.ptr = NULL;
>> - return 0;
>> - }
>> -
>> - if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
>> - return -EINVAL;
>> -
>> - amdgpu_asic_init_doorbell_index(adev);
>> -
>> - /* doorbell bar mapping */
>> - adev->doorbell.base = pci_resource_start(adev->pdev, 2);
>> - adev->doorbell.size = pci_resource_len(adev->pdev, 2);
>> -
>> - if (adev->enable_mes) {
>> - adev->doorbell.num_kernel_doorbells =
>> - adev->doorbell.size / sizeof(u32);
>> - } else {
>> - adev->doorbell.num_kernel_doorbells =
>> - min_t(u32, adev->doorbell.size / sizeof(u32),
>> - adev->doorbell_index.max_assignment+1);
>> - if (adev->doorbell.num_kernel_doorbells == 0)
>> - return -EINVAL;
>> -
>> - /* For Vega, reserve and map two pages on doorbell BAR since
>> SDMA
>> - * paging queue doorbell use the second page. The
>> - * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
>> - * doorbells are in the first page. So with paging queue
>> enabled,
>> - * the max num_kernel_doorbells should + 1 page (0x400 in dword)
>> - */
>> - if (adev->asic_type >= CHIP_VEGA10)
>> - adev->doorbell.num_kernel_doorbells += 0x400;
>> - }
>> -
>> - adev->doorbell.ptr = ioremap(adev->doorbell.base,
>> - adev->doorbell.num_kernel_doorbells *
>> - sizeof(u32));
>> - if (adev->doorbell.ptr == NULL)
>> - return -ENOMEM;
>> -
>> - return 0;
>> -}
>> -
>> -/**
>> - * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>> - *
>> - * @adev: amdgpu_device pointer
>> - *
>> - * Tear down doorbell driver information (CIK)
>> - */
>> -static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
>> -{
>> - iounmap(adev->doorbell.ptr);
>> - adev->doorbell.ptr = NULL;
>> -}
>> -
>> -
>> -
>> /*
>> * amdgpu_device_wb_*()
>> * Writeback is the method by which the GPU updates special pages in memory
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> index 6064943a1b53..f9c3b77bf65d 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>> @@ -306,6 +306,28 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev,
>> u32 index, u32 v);
>> u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
>> void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
>>
>> +/*
>> + * GPU doorbell aperture helpers function.
>> + */
>> +/**
>> + * amdgpu_device_doorbell_init - Init doorbell driver information.
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * Init doorbell driver information (CIK)
>> + * Returns 0 on success, error on failure.
>> + */
>> +int amdgpu_device_doorbell_init(struct amdgpu_device *adev);
>> +
>> +/**
>> + * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * Tear down doorbell driver information (CIK)
>> + */
>> +void amdgpu_device_doorbell_fini(struct amdgpu_device *adev);
>
> I might be wrong, but having kerneldoc on both the declaration and the
> implementation is forbidden I think.
>
> We usually have it on the implementation side only.
Yes, that is correct--implementation side only as that's where
the actual code can be found.
>
> Christian.
>
>> +
>> #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
>> #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
>> #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>> new file mode 100644
>> index 000000000000..2206926ba289
>> --- /dev/null
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>> @@ -0,0 +1,186 @@
>> +/*
Add an SPDX licence identifier as the very first line:
//SPDX-License-Identifier: <value>
with some appropriate value, maybe "GPL-2.0-or-later" (no quotes when added).
Regards,
Luben
>> + * Copyright 2023 Advanced Micro Devices, Inc.
>> + *
>> + * Permission is hereby granted, free of charge, to any person obtaining a
>> + * copy of this software and associated documentation files (the
>> "Software"),
>> + * to deal in the Software without restriction, including without limitation
>> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
>> + * and/or sell copies of the Software, and to permit persons to whom the
>> + * Software is furnished to do so, subject to the following conditions:
>> + *
>> + * The above copyright notice and this permission notice shall be included
>> in
>> + * all copies or substantial portions of the Software.
>> + *
>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
>> OR
>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
>> + * OTHER DEALINGS IN THE SOFTWARE.
>> + *
>> + */
>> +
>> +#include "amdgpu.h"
>> +
>> +/**
>> + * amdgpu_mm_rdoorbell - read a doorbell dword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + *
>> + * Returns the value in the doorbell aperture at the
>> + * requested doorbell index (CIK).
>> + */
>> +u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
>> +{
>> + if (amdgpu_device_skip_hw_access(adev))
>> + return 0;
>> +
>> + if (index < adev->doorbell.num_kernel_doorbells) {
>> + return readl(adev->doorbell.ptr + index);
>> + } else {
>> + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>> + return 0;
>> + }
>> +}
>> +
>> +/**
>> + * amdgpu_mm_wdoorbell - write a doorbell dword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + * @v: value to write
>> + *
>> + * Writes @v to the doorbell aperture at the
>> + * requested doorbell index (CIK).
>> + */
>> +void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
>> +{
>> + if (amdgpu_device_skip_hw_access(adev))
>> + return;
>> +
>> + if (index < adev->doorbell.num_kernel_doorbells) {
>> + writel(v, adev->doorbell.ptr + index);
>> + } else {
>> + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>> + }
>> +}
>> +
>> +/**
>> + * amdgpu_mm_rdoorbell64 - read a doorbell Qword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + *
>> + * Returns the value in the doorbell aperture at the
>> + * requested doorbell index (VEGA10+).
>> + */
>> +u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
>> +{
>> + if (amdgpu_device_skip_hw_access(adev))
>> + return 0;
>> +
>> + if (index < adev->doorbell.num_kernel_doorbells) {
>> + return atomic64_read((atomic64_t *)(adev->doorbell.ptr +
>> index));
>> + } else {
>> + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>> + return 0;
>> + }
>> +}
>> +
>> +/**
>> + * amdgpu_mm_wdoorbell64 - write a doorbell Qword
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @index: doorbell index
>> + * @v: value to write
>> + *
>> + * Writes @v to the doorbell aperture at the
>> + * requested doorbell index (VEGA10+).
>> + */
>> +void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
>> +{
>> + if (amdgpu_device_skip_hw_access(adev))
>> + return;
>> +
>> + if (index < adev->doorbell.num_kernel_doorbells) {
>> + atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>> + } else {
>> + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>> + }
>> +}
>> +
>> +/*
>> + * GPU doorbell aperture helpers function.
>> + */
>> +/**
>> + * amdgpu_device_doorbell_init - Init doorbell driver information.
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * Init doorbell driver information (CIK)
>> + * Returns 0 on success, error on failure.
>> + */
>> +int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
>> +{
>> +
>> + /* No doorbell on SI hardware generation */
>> + if (adev->asic_type < CHIP_BONAIRE) {
>> + adev->doorbell.base = 0;
>> + adev->doorbell.size = 0;
>> + adev->doorbell.num_kernel_doorbells = 0;
>> + adev->doorbell.ptr = NULL;
>> + return 0;
>> + }
>> +
>> + if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
>> + return -EINVAL;
>> +
>> + amdgpu_asic_init_doorbell_index(adev);
>> +
>> + /* doorbell bar mapping */
>> + adev->doorbell.base = pci_resource_start(adev->pdev, 2);
>> + adev->doorbell.size = pci_resource_len(adev->pdev, 2);
>> +
>> + if (adev->enable_mes) {
>> + adev->doorbell.num_kernel_doorbells =
>> + adev->doorbell.size / sizeof(u32);
>> + } else {
>> + adev->doorbell.num_kernel_doorbells =
>> + min_t(u32, adev->doorbell.size / sizeof(u32),
>> + adev->doorbell_index.max_assignment+1);
>> + if (adev->doorbell.num_kernel_doorbells == 0)
>> + return -EINVAL;
>> +
>> + /* For Vega, reserve and map two pages on doorbell BAR since
>> SDMA
>> + * paging queue doorbell use the second page. The
>> + * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
>> + * doorbells are in the first page. So with paging queue
>> enabled,
>> + * the max num_kernel_doorbells should + 1 page (0x400 in dword)
>> + */
>> + if (adev->asic_type >= CHIP_VEGA10)
>> + adev->doorbell.num_kernel_doorbells += 0x400;
>> + }
>> +
>> + adev->doorbell.ptr = ioremap(adev->doorbell.base,
>> + adev->doorbell.num_kernel_doorbells *
>> + sizeof(u32));
>> + if (adev->doorbell.ptr == NULL)
>> + return -ENOMEM;
>> +
>> + return 0;
>> +}
>> +
>> +/**
>> + * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * Tear down doorbell driver information (CIK)
>> + */
>> +void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
>> +{
>> + iounmap(adev->doorbell.ptr);
>> + adev->doorbell.ptr = NULL;
>> +}
>
