Re: [RFC 1/4] drm/nouveau: Add support for basic clockgating on Kepler1

[Martin Peres]

On 16/01/18 00:06, Lyude Paul wrote:
> This adds support for enabling automatic clockgating on nvidia GPUs for
> Kepler1, referred to as "CG" throughout the driver. This is one of two
> powersaving levels that Kepler1 supports.

Thanks a lot for all this work! It was long overdue and it is nice to
see the project finally getting to an end, after passing into so many hands!

> 
> This introduces two therm helpers for controlling basic clockgating:
>   nvkm_therm_clkgate_enable() - enables clockgating through
>   CG_CTRL, done after initializing the GPU fully
>   nvkm_therm_clkgate_fini() - prepares clockgating for suspend or
>   driver unload
> 
> As well, we add the nouveau kernel config parameter NvPmEnableGating,
> which can be set to the highest level of clockgating (in this case, we
> only have CG) the user desires to enable. Since we've only had limited
> testing on this thus far, we disable this by default.

I am not sure I understand the purpose of this level here. As far as I
understand, you only have per-engine control whether you want to enable
CG or not. What you call BLCG and SLCG levels just mean "don't use the
boot values, but rather use our values (taken from nvidia)".

Now, here comes the nasty part: NVIDIA only ever validated the boot
values (I guess they are extremely safe ones), and the optimised values
(the ones coming from your patch 2, 3, and 4 along with the level 3.

I think introducing a single parameter that controls both CG, PG, and
automatic reclocking would be safer. For CG and PG, it would be a
all-or-nothing (either boot values, or everything like nvidia).

> 
> A lot of this code was originally going to be based off of fermi;
> however it turns out that while Fermi's the first line of GPUs that
> introduced this kind of power saving, Fermi requires more fine tuned
> control of the CG_CTRL registers from the driver while reclocking that
> we don't entirely understand yet.
> 
> For the simple parts we will be sharing with Fermi for certain however,
> we at least add those into a new subdev/therm/gf100.h header.
> 
> Signed-off-by: Lyude Paul 
> ---
>  .../gpu/drm/nouveau/include/nvkm/subdev/therm.h|  10 ++
>  drivers/gpu/drm/nouveau/nvkm/engine/device/base.c  |  17 +--
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild   |   1 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c   |  72 +--
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h  |  35 ++
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c  |   8 +-
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c  | 135 
> +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h  |  56 +
>  drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h   |  15 ++-
>  9 files changed, 328 insertions(+), 21 deletions(-)
>  create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h
>  create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c
>  create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h
> 
> diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h 
> b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
> index b1ac47eb786e..a9204c09975b 100644
> --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
> +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
> @@ -46,6 +46,11 @@ enum nvkm_therm_attr_type {
>   NVKM_THERM_ATTR_THRS_SHUTDOWN_HYST = 17,
>  };
>  
> +enum nvkm_therm_clkgate_level {
> + NVKM_THERM_CLKGATE_NONE = 0,
> + NVKM_THERM_CLKGATE_CG, /* basic clockgating */
> +};
> +
>  struct nvkm_therm {
>   const struct nvkm_therm_func *func;
>   struct nvkm_subdev subdev;
> @@ -85,17 +90,22 @@ struct nvkm_therm {
>  
>   int (*attr_get)(struct nvkm_therm *, enum nvkm_therm_attr_type);
>   int (*attr_set)(struct nvkm_therm *, enum nvkm_therm_attr_type, int);
> +
> + enum nvkm_therm_clkgate_level clkgate_level;
>  };
>  
>  int nvkm_therm_temp_get(struct nvkm_therm *);
>  int nvkm_therm_fan_sense(struct nvkm_therm *);
>  int nvkm_therm_cstate(struct nvkm_therm *, int, int);
> +void nvkm_therm_clkgate_enable(struct nvkm_therm *);
> +void nvkm_therm_clkgate_fini(struct nvkm_therm *, bool);
>  
>  int nv40_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int nv50_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int g84_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int gt215_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int gf119_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
> +int gk104_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int gm107_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int gm200_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
>  int gp100_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
> diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c 
> b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
> index 

[RFC 1/4] drm/nouveau: Add support for basic clockgating on Kepler1

[Lyude Paul]

This adds support for enabling automatic clockgating on nvidia GPUs for
Kepler1, referred to as "CG" throughout the driver. This is one of two
powersaving levels that Kepler1 supports.

This introduces two therm helpers for controlling basic clockgating:
nvkm_therm_clkgate_enable() - enables clockgating through
CG_CTRL, done after initializing the GPU fully
nvkm_therm_clkgate_fini() - prepares clockgating for suspend or
driver unload

As well, we add the nouveau kernel config parameter NvPmEnableGating,
which can be set to the highest level of clockgating (in this case, we
only have CG) the user desires to enable. Since we've only had limited
testing on this thus far, we disable this by default.

A lot of this code was originally going to be based off of fermi;
however it turns out that while Fermi's the first line of GPUs that
introduced this kind of power saving, Fermi requires more fine tuned
control of the CG_CTRL registers from the driver while reclocking that
we don't entirely understand yet.

For the simple parts we will be sharing with Fermi for certain however,
we at least add those into a new subdev/therm/gf100.h header.

Signed-off-by: Lyude Paul 
---
 .../gpu/drm/nouveau/include/nvkm/subdev/therm.h|  10 ++
 drivers/gpu/drm/nouveau/nvkm/engine/device/base.c  |  17 +--
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild   |   1 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c   |  72 +--
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h  |  35 ++
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c  |   8 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c  | 135 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h  |  56 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h   |  15 ++-
 9 files changed, 328 insertions(+), 21 deletions(-)
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h 
b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
index b1ac47eb786e..a9204c09975b 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
@@ -46,6 +46,11 @@ enum nvkm_therm_attr_type {
NVKM_THERM_ATTR_THRS_SHUTDOWN_HYST = 17,
 };
 
+enum nvkm_therm_clkgate_level {
+   NVKM_THERM_CLKGATE_NONE = 0,
+   NVKM_THERM_CLKGATE_CG, /* basic clockgating */
+};
+
 struct nvkm_therm {
const struct nvkm_therm_func *func;
struct nvkm_subdev subdev;
@@ -85,17 +90,22 @@ struct nvkm_therm {
 
int (*attr_get)(struct nvkm_therm *, enum nvkm_therm_attr_type);
int (*attr_set)(struct nvkm_therm *, enum nvkm_therm_attr_type, int);
+
+   enum nvkm_therm_clkgate_level clkgate_level;
 };
 
 int nvkm_therm_temp_get(struct nvkm_therm *);
 int nvkm_therm_fan_sense(struct nvkm_therm *);
 int nvkm_therm_cstate(struct nvkm_therm *, int, int);
+void nvkm_therm_clkgate_enable(struct nvkm_therm *);
+void nvkm_therm_clkgate_fini(struct nvkm_therm *, bool);
 
 int nv40_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int nv50_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int g84_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gt215_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gf119_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
+int gk104_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gm107_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gm200_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gp100_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c 
b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 00eeaaffeae5..6c5f966c66ad 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -28,6 +28,7 @@
 #include 
 
 #include 
+#include 
 
 static DEFINE_MUTEX(nv_devices_mutex);
 static LIST_HEAD(nv_devices);
@@ -1682,7 +1683,7 @@ nve4_chipset = {
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk104_pmu_new,
-   .therm = gf119_therm_new,
+   .therm = gk104_therm_new,
.timer = nv41_timer_new,
.top = gk104_top_new,
.volt = gk104_volt_new,
@@ -1721,7 +1722,7 @@ nve6_chipset = {
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk104_pmu_new,
-   .therm = gf119_therm_new,
+   .therm = gk104_therm_new,
.timer = nv41_timer_new,
.top = gk104_top_new,
.volt = gk104_volt_new,
@@ -1760,7 +1761,7 @@ nve7_chipset = {
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk104_pmu_new,
-   .therm = gf119_therm_new,
+   .therm