[PATCH v9 02/14] drm/amdgpu: add usermode queue base code

2024-04-26 Thread Shashank Sharma 
This patch adds skeleton code for amdgpu usermode queue.
It contains:
- A new files with init functions of usermode queues.
- A queue context manager in driver private data.

V1: Worked on design review comments from RFC patch series:
(https://patchwork.freedesktop.org/series/112214/)
- Alex: Keep a list of queues, instead of single queue per process.
- Christian: Use the queue manager instead of global ptrs,
   Don't keep the queue structure in amdgpu_ctx

V2:
 - Reformatted code, split the big patch into two

V3:
- Integration with doorbell manager

V4:
- Align the structure member names to the largest member's column
  (Luben)
- Added SPDX license (Luben)

V5:
- Do not add amdgpu.h in amdgpu_userqueue.h (Christian).
- Move struct amdgpu_userq_mgr into amdgpu_userqueue.h (Christian).

V6: Rebase
V9: Rebase

Cc: Alex Deucher 
Cc: Christian Koenig 
Reviewed-by: Christian König 
Signed-off-by: Shashank Sharma 
---
 drivers/gpu/drm/amd/amdgpu/Makefile   |  2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu.h   |  3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c   |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c   |  6 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 40 
 .../gpu/drm/amd/include/amdgpu_userqueue.h| 61 +++
 6 files changed, 113 insertions(+)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
 create mode 100644 drivers/gpu/drm/amd/include/amdgpu_userqueue.h

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
b/drivers/gpu/drm/amd/amdgpu/Makefile
index 4536c8ad0e11..05a2d1714070 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -260,6 +260,8 @@ amdgpu-y += \
 # add amdkfd interfaces
 amdgpu-y += amdgpu_amdkfd.o
 
+# add usermode queue
+amdgpu-y += amdgpu_userqueue.o
 
 ifneq ($(CONFIG_HSA_AMD),)
 AMDKFD_PATH := ../amdkfd
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h 
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index b3b84647207e..4ca14b02668b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -112,6 +112,7 @@
 #include "amdgpu_xcp.h"
 #include "amdgpu_seq64.h"
 #include "amdgpu_reg_state.h"
+#include "amdgpu_userqueue.h"
 
 #define MAX_GPU_INSTANCE   64
 
@@ -477,6 +478,7 @@ struct amdgpu_fpriv {
struct mutexbo_list_lock;
struct idr  bo_list_handles;
struct amdgpu_ctx_mgr   ctx_mgr;
+   struct amdgpu_userq_mgr userq_mgr;
/** GPU partition selection */
uint32_txcp_id;
 };
@@ -1039,6 +1041,7 @@ struct amdgpu_device {
boolenable_mes_kiq;
struct amdgpu_mes   mes;
struct amdgpu_mqd   mqds[AMDGPU_HW_IP_NUM];
+   const struct amdgpu_userq_funcs *userq_funcs[AMDGPU_HW_IP_NUM];
 
/* df */
struct amdgpu_dfdf;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index e4277298cf1a..374970984a61 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -50,6 +50,7 @@
 #include "amdgpu_reset.h"
 #include "amdgpu_sched.h"
 #include "amdgpu_xgmi.h"
+#include "amdgpu_userqueue.h"
 #include "../amdxcp/amdgpu_xcp_drv.h"
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index a2df3025a754..d78b06af834e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -44,6 +44,7 @@
 #include "amdgpu_display.h"
 #include "amdgpu_ras.h"
 #include "amd_pcie.h"
+#include "amdgpu_userqueue.h"
 
 void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
 {
@@ -1388,6 +1389,10 @@ int amdgpu_driver_open_kms(struct drm_device *dev, 
struct drm_file *file_priv)
 
amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
 
+   r = amdgpu_userq_mgr_init(&fpriv->userq_mgr, adev);
+   if (r)
+   DRM_WARN("Can't setup usermode queues, use legacy workload 
submission only\n");
+
file_priv->driver_priv = fpriv;
goto out_suspend;
 
@@ -1457,6 +1462,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 
amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
amdgpu_vm_fini(adev, &fpriv->vm);
+   amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
 
if (pasid)
amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
new file mode 100644
index ..effc0c7c02cf
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including wi

Re: [PATCH v9 02/14] drm/amdgpu: add usermode queue base code

2024-05-01 Thread Alex Deucher 
On Fri, Apr 26, 2024 at 9:57 AM Shashank Sharma  wrote:
>
> This patch adds skeleton code for amdgpu usermode queue.
> It contains:
> - A new files with init functions of usermode queues.
> - A queue context manager in driver private data.
>
> V1: Worked on design review comments from RFC patch series:
> (https://patchwork.freedesktop.org/series/112214/)
> - Alex: Keep a list of queues, instead of single queue per process.
> - Christian: Use the queue manager instead of global ptrs,
>Don't keep the queue structure in amdgpu_ctx
>
> V2:
>  - Reformatted code, split the big patch into two
>
> V3:
> - Integration with doorbell manager
>
> V4:
> - Align the structure member names to the largest member's column
>   (Luben)
> - Added SPDX license (Luben)
>
> V5:
> - Do not add amdgpu.h in amdgpu_userqueue.h (Christian).
> - Move struct amdgpu_userq_mgr into amdgpu_userqueue.h (Christian).
>
> V6: Rebase
> V9: Rebase
>
> Cc: Alex Deucher 
> Cc: Christian Koenig 
> Reviewed-by: Christian König 
> Signed-off-by: Shashank Sharma 

Reviewed-by: Alex Deucher 

> ---
>  drivers/gpu/drm/amd/amdgpu/Makefile   |  2 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h   |  3 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c   |  1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c   |  6 ++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 40 
>  .../gpu/drm/amd/include/amdgpu_userqueue.h| 61 +++
>  6 files changed, 113 insertions(+)
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
>  create mode 100644 drivers/gpu/drm/amd/include/amdgpu_userqueue.h
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4536c8ad0e11..05a2d1714070 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -260,6 +260,8 @@ amdgpu-y += \
>  # add amdkfd interfaces
>  amdgpu-y += amdgpu_amdkfd.o
>
> +# add usermode queue
> +amdgpu-y += amdgpu_userqueue.o
>
>  ifneq ($(CONFIG_HSA_AMD),)
>  AMDKFD_PATH := ../amdkfd
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index b3b84647207e..4ca14b02668b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -112,6 +112,7 @@
>  #include "amdgpu_xcp.h"
>  #include "amdgpu_seq64.h"
>  #include "amdgpu_reg_state.h"
> +#include "amdgpu_userqueue.h"
>
>  #define MAX_GPU_INSTANCE   64
>
> @@ -477,6 +478,7 @@ struct amdgpu_fpriv {
> struct mutexbo_list_lock;
> struct idr  bo_list_handles;
> struct amdgpu_ctx_mgr   ctx_mgr;
> +   struct amdgpu_userq_mgr userq_mgr;
> /** GPU partition selection */
> uint32_txcp_id;
>  };
> @@ -1039,6 +1041,7 @@ struct amdgpu_device {
> boolenable_mes_kiq;
> struct amdgpu_mes   mes;
> struct amdgpu_mqd   mqds[AMDGPU_HW_IP_NUM];
> +   const struct amdgpu_userq_funcs *userq_funcs[AMDGPU_HW_IP_NUM];
>
> /* df */
> struct amdgpu_dfdf;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index e4277298cf1a..374970984a61 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -50,6 +50,7 @@
>  #include "amdgpu_reset.h"
>  #include "amdgpu_sched.h"
>  #include "amdgpu_xgmi.h"
> +#include "amdgpu_userqueue.h"
>  #include "../amdxcp/amdgpu_xcp_drv.h"
>
>  /*
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index a2df3025a754..d78b06af834e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -44,6 +44,7 @@
>  #include "amdgpu_display.h"
>  #include "amdgpu_ras.h"
>  #include "amd_pcie.h"
> +#include "amdgpu_userqueue.h"
>
>  void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
>  {
> @@ -1388,6 +1389,10 @@ int amdgpu_driver_open_kms(struct drm_device *dev, 
> struct drm_file *file_priv)
>
> amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
>
> +   r = amdgpu_userq_mgr_init(&fpriv->userq_mgr, adev);
> +   if (r)
> +   DRM_WARN("Can't setup usermode queues, use legacy workload 
> submission only\n");
> +
> file_priv->driver_priv = fpriv;
> goto out_suspend;
>
> @@ -1457,6 +1462,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
>
> amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
> amdgpu_vm_fini(adev, &fpriv->vm);
> +   amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
>
> if (pasid)
> amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
> new file mode 100644
> index ..effc0c7c02cf
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu