Re: [PATCH 04/21] drm/amdgpu: Add GFXv9 kfd2kgd interface functions

[Oded Gabbay]

On Wed, May 16, 2018 at 1:08 AM, Felix Kuehling  wrote:
> On 2018-05-15 05:41 AM, Dave Airlie wrote:
>>> +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
>>> +   uint32_t queue_id, uint32_t __user *wptr,
>>> +   uint32_t wptr_shift, uint32_t wptr_mask,
>>> +   struct mm_struct *mm)
>>> +{
>>> +   struct amdgpu_device *adev = get_amdgpu_device(kgd);
>>> +   struct v9_mqd *m;
>>> +   uint32_t *mqd_hqd;
>>> +   uint32_t reg, hqd_base, data;
>>> +
>>> +   m = get_mqd(mqd);
>>> +
>>> +   acquire_queue(kgd, pipe_id, queue_id);
>>> +
>>> +   /* HIQ is set during driver init period with vmid set to 0*/
>>> +   if (m->cp_hqd_vmid == 0) {
>>> +   uint32_t value, mec, pipe;
>>> +
>>> +   mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
>>> +   pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
>>> +
>>> +   pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
>>> +   mec, pipe, queue_id);
>>> +   value = RREG32(SOC15_REG_OFFSET(GC, 0, 
>>> mmRLC_CP_SCHEDULERS));
>>> +   value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
>>> +   ((mec << 5) | (pipe << 3) | queue_id | 0x80));
>>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
>>> +   }
>>> +
>>> +   /* HQD registers extend from CP_MQD_BASE_ADDR to 
>>> CP_HQD_EOP_WPTR_MEM. */
>>> +   mqd_hqd = >cp_mqd_base_addr_lo;
>>> +   hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
>>> +
>>> +   for (reg = hqd_base;
>>> +reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
>>> +   WREG32(reg, mqd_hqd[reg - hqd_base]);
>>> +
>>> +
>>> +   /* Activate doorbell logic before triggering WPTR poll. */
>>> +   data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
>>> +CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
>>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
>>> +
>>> +   if (wptr) {
>>> +   /* Don't read wptr with get_user because the user
>>> +* context may not be accessible (if this function
>>> +* runs in a work queue). Instead trigger a one-shot
>>> +* polling read from memory in the CP. This assumes
>>> +* that wptr is GPU-accessible in the queue's VMID via
>>> +* ATC or SVM. WPTR==RPTR before starting the poll so
>>> +* the CP starts fetching new commands from the right
>>> +* place.
>>> +*
>>> +* Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
>>> +* tricky. Assume that the queue didn't overflow. The
>>> +* number of valid bits in the 32-bit RPTR depends on
>>> +* the queue size. The remaining bits are taken from
>>> +* the saved 64-bit WPTR. If the WPTR wrapped, add the
>>> +* queue size.
>>> +*/
>>> +   uint32_t queue_size =
>>> +   2 << REG_GET_FIELD(m->cp_hqd_pq_control,
>>> +  CP_HQD_PQ_CONTROL, QUEUE_SIZE);
>>> +   uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 
>>> 1);
>>> +
>>> +   if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < 
>>> guessed_wptr)
>>> +   guessed_wptr += queue_size;
>>> +   guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
>>> +   guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
>>> +
>>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
>>> +  lower_32_bits(guessed_wptr));
>>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
>>> +  upper_32_bits(guessed_wptr));
>>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
>>> +  lower_32_bits((uint64_t)wptr));
>>> +   WREG32(SOC15_REG_OFFSET(GC, 0, 
>>> mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
>>> +  upper_32_bits((uint64_t)wptr));
>>  CC [M]  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.o
>> In file included from
>> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:30:0:
>> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:
>> In function ‘kgd_hqd_load’:
>> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:473:24:
>> warning: cast from pointer to integer of different size
>> [-Wpointer-to-int-cast]
>>   lower_32_bits((uint64_t)wptr));
>> ^
>> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu.h:1666:53:
>> note: in definition of macro ‘WREG32’
>>  #define WREG32(reg, v) 

Re: [PATCH 04/21] drm/amdgpu: Add GFXv9 kfd2kgd interface functions

[Felix Kuehling]

On 2018-05-15 05:41 AM, Dave Airlie wrote:
>> +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
>> +   uint32_t queue_id, uint32_t __user *wptr,
>> +   uint32_t wptr_shift, uint32_t wptr_mask,
>> +   struct mm_struct *mm)
>> +{
>> +   struct amdgpu_device *adev = get_amdgpu_device(kgd);
>> +   struct v9_mqd *m;
>> +   uint32_t *mqd_hqd;
>> +   uint32_t reg, hqd_base, data;
>> +
>> +   m = get_mqd(mqd);
>> +
>> +   acquire_queue(kgd, pipe_id, queue_id);
>> +
>> +   /* HIQ is set during driver init period with vmid set to 0*/
>> +   if (m->cp_hqd_vmid == 0) {
>> +   uint32_t value, mec, pipe;
>> +
>> +   mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
>> +   pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
>> +
>> +   pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
>> +   mec, pipe, queue_id);
>> +   value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
>> +   value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
>> +   ((mec << 5) | (pipe << 3) | queue_id | 0x80));
>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
>> +   }
>> +
>> +   /* HQD registers extend from CP_MQD_BASE_ADDR to 
>> CP_HQD_EOP_WPTR_MEM. */
>> +   mqd_hqd = >cp_mqd_base_addr_lo;
>> +   hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
>> +
>> +   for (reg = hqd_base;
>> +reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
>> +   WREG32(reg, mqd_hqd[reg - hqd_base]);
>> +
>> +
>> +   /* Activate doorbell logic before triggering WPTR poll. */
>> +   data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
>> +CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
>> +
>> +   if (wptr) {
>> +   /* Don't read wptr with get_user because the user
>> +* context may not be accessible (if this function
>> +* runs in a work queue). Instead trigger a one-shot
>> +* polling read from memory in the CP. This assumes
>> +* that wptr is GPU-accessible in the queue's VMID via
>> +* ATC or SVM. WPTR==RPTR before starting the poll so
>> +* the CP starts fetching new commands from the right
>> +* place.
>> +*
>> +* Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
>> +* tricky. Assume that the queue didn't overflow. The
>> +* number of valid bits in the 32-bit RPTR depends on
>> +* the queue size. The remaining bits are taken from
>> +* the saved 64-bit WPTR. If the WPTR wrapped, add the
>> +* queue size.
>> +*/
>> +   uint32_t queue_size =
>> +   2 << REG_GET_FIELD(m->cp_hqd_pq_control,
>> +  CP_HQD_PQ_CONTROL, QUEUE_SIZE);
>> +   uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
>> +
>> +   if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
>> +   guessed_wptr += queue_size;
>> +   guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
>> +   guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
>> +
>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
>> +  lower_32_bits(guessed_wptr));
>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
>> +  upper_32_bits(guessed_wptr));
>> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
>> +  lower_32_bits((uint64_t)wptr));
>> +   WREG32(SOC15_REG_OFFSET(GC, 0, 
>> mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
>> +  upper_32_bits((uint64_t)wptr));
>  CC [M]  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.o
> In file included from
> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:30:0:
> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:
> In function ‘kgd_hqd_load’:
> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:473:24:
> warning: cast from pointer to integer of different size
> [-Wpointer-to-int-cast]
>   lower_32_bits((uint64_t)wptr));
> ^
> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu.h:1666:53:
> note: in definition of macro ‘WREG32’
>  #define WREG32(reg, v) amdgpu_mm_wreg(adev, (reg), (v), 0)
>  ^
> /home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:473:10:
> note: 

Re: [PATCH 04/21] drm/amdgpu: Add GFXv9 kfd2kgd interface functions

[Dave Airlie]

> +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
> +   uint32_t queue_id, uint32_t __user *wptr,
> +   uint32_t wptr_shift, uint32_t wptr_mask,
> +   struct mm_struct *mm)
> +{
> +   struct amdgpu_device *adev = get_amdgpu_device(kgd);
> +   struct v9_mqd *m;
> +   uint32_t *mqd_hqd;
> +   uint32_t reg, hqd_base, data;
> +
> +   m = get_mqd(mqd);
> +
> +   acquire_queue(kgd, pipe_id, queue_id);
> +
> +   /* HIQ is set during driver init period with vmid set to 0*/
> +   if (m->cp_hqd_vmid == 0) {
> +   uint32_t value, mec, pipe;
> +
> +   mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
> +   pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
> +
> +   pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
> +   mec, pipe, queue_id);
> +   value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
> +   value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
> +   ((mec << 5) | (pipe << 3) | queue_id | 0x80));
> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
> +   }
> +
> +   /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. 
> */
> +   mqd_hqd = >cp_mqd_base_addr_lo;
> +   hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
> +
> +   for (reg = hqd_base;
> +reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
> +   WREG32(reg, mqd_hqd[reg - hqd_base]);
> +
> +
> +   /* Activate doorbell logic before triggering WPTR poll. */
> +   data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
> +CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
> +
> +   if (wptr) {
> +   /* Don't read wptr with get_user because the user
> +* context may not be accessible (if this function
> +* runs in a work queue). Instead trigger a one-shot
> +* polling read from memory in the CP. This assumes
> +* that wptr is GPU-accessible in the queue's VMID via
> +* ATC or SVM. WPTR==RPTR before starting the poll so
> +* the CP starts fetching new commands from the right
> +* place.
> +*
> +* Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
> +* tricky. Assume that the queue didn't overflow. The
> +* number of valid bits in the 32-bit RPTR depends on
> +* the queue size. The remaining bits are taken from
> +* the saved 64-bit WPTR. If the WPTR wrapped, add the
> +* queue size.
> +*/
> +   uint32_t queue_size =
> +   2 << REG_GET_FIELD(m->cp_hqd_pq_control,
> +  CP_HQD_PQ_CONTROL, QUEUE_SIZE);
> +   uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
> +
> +   if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
> +   guessed_wptr += queue_size;
> +   guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
> +   guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
> +
> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
> +  lower_32_bits(guessed_wptr));
> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
> +  upper_32_bits(guessed_wptr));
> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
> +  lower_32_bits((uint64_t)wptr));
> +   WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
> +  upper_32_bits((uint64_t)wptr));

 CC [M]  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.o
In file included from
/home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:30:0:
/home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:
In function ‘kgd_hqd_load’:
/home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:473:24:
warning: cast from pointer to integer of different size
[-Wpointer-to-int-cast]
  lower_32_bits((uint64_t)wptr));
^
/home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu.h:1666:53:
note: in definition of macro ‘WREG32’
 #define WREG32(reg, v) amdgpu_mm_wreg(adev, (reg), (v), 0)
 ^
/home/airlied/devel/kernel/dim/src/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c:473:10:
note: in expansion of macro ‘lower_32_bits’
  lower_32_bits((uint64_t)wptr));
  ^

[PATCH 04/21] drm/amdgpu: Add GFXv9 kfd2kgd interface functions

[Felix Kuehling]

Signed-off-by: John Bridgman 
Signed-off-by: Shaoyun Liu 
Signed-off-by: Jay Cornwall 
Signed-off-by: Yong Zhao 
Signed-off-by: Felix Kuehling 
---
 MAINTAINERS   |1 +
 drivers/gpu/drm/amd/amdgpu/Makefile   |3 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c|4 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h|1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 1043 +
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c |1 +
 6 files changed, 1052 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c

diff --git a/MAINTAINERS b/MAINTAINERS
index 6804170..9bfb765 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -766,6 +766,7 @@ F:  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
 F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
 F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
 F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
 F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
 F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
 F: drivers/gpu/drm/amd/amdkfd/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
b/drivers/gpu/drm/amd/amdgpu/Makefile
index 2ca2b51..f300202 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -130,7 +130,8 @@ amdgpu-y += \
 amdgpu_amdkfd.o \
 amdgpu_amdkfd_fence.o \
 amdgpu_amdkfd_gpuvm.o \
-amdgpu_amdkfd_gfx_v8.o
+amdgpu_amdkfd_gfx_v8.o \
+amdgpu_amdkfd_gfx_v9.o
 
 # add cgs
 amdgpu-y += amdgpu_cgs.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 4d36203..fcd10db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -92,6 +92,10 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
case CHIP_POLARIS11:
kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
break;
+   case CHIP_VEGA10:
+   case CHIP_RAVEN:
+   kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions();
+   break;
default:
dev_dbg(adev->dev, "kfd not supported on this ASIC\n");
return;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index c3024b1..12367a9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -122,6 +122,7 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum 
kgd_engine_type engine,
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
 
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
new file mode 100644
index 000..8f37991
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -0,0 +1,1043 @@
+/*
+ * Copyright 2014-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#define pr_fmt(fmt) "kfd2kgd: " fmt
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
+#include "amdgpu_ucode.h"
+#include "soc15_hw_ip.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "vega10_enum.h"
+#include "sdma0/sdma0_4_0_offset.h"
+#include "sdma0/sdma0_4_0_sh_mask.h"
+#include "sdma1/sdma1_4_0_offset.h"
+#include "sdma1/sdma1_4_0_sh_mask.h"
+#include "athub/athub_1_0_offset.h"
+#include "athub/athub_1_0_sh_mask.h"
+#include