[PATCH V5] drm/amdgpu: Surface svm_default_granularity, a RW module parameter
Enables users to update SVM's default granularity, used in
buffer migration and handling of recoverable page faults.
Param value is set in terms of log(numPages(buffer)),
e.g. 9 for a 2 MIB buffer
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 17 +
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 6 ++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 23 +++
4 files changed, 39 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e8c284aea1f2..8eb934af02f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -237,6 +237,7 @@ extern int sched_policy;
extern bool debug_evictions;
extern bool no_system_mem_limit;
extern int halt_if_hws_hang;
+extern uint amdgpu_svm_default_granularity;
#else
static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b9529948f2b2..442039436cb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -169,6 +169,16 @@ uint amdgpu_sdma_phase_quantum = 32;
char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
bool enforce_isolation;
+
+/* Specifies the default granularity for SVM, used in buffer
+ * migration and restoration of backing memory when handling
+ * recoverable page faults.
+ *
+ * The value is given as log(numPages(buffer)); for a 2 MiB
+ * buffer it computes to be 9
+ */
+uint amdgpu_svm_default_granularity = 9;
+
/*
* OverDrive(bit 14) disabled by default
* GFX DCS(bit 19) disabled by default
@@ -320,6 +330,13 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
+/**
+ * DOC: svm_default_granularity (uint)
+ * Used in buffer migration and handling of recoverable page faults
+ */
+MODULE_PARM_DESC(svm_default_granularity, "SVM's default granularity in
log(2^Pages), default 9 = 2^9 = 2 MiB");
+module_param_named(svm_default_granularity, amdgpu_svm_default_granularity,
uint, 0644);
+
/**
* DOC: lockup_timeout (string)
* Set GPU scheduler timeout value in ms.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9ae9abc6eb43..d6530febabad 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -868,6 +868,12 @@ struct svm_range_list {
struct task_struct *faulting_task;
/* check point ts decides if page fault recovery need be dropped */
uint64_tcheckpoint_ts[MAX_GPU_INSTANCE];
+
+ /* Default granularity to use in buffer migration
+* and restoration of backing memory while handling
+* recoverable page faults
+*/
+ uint8_t default_granularity;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index b44dec90969f..2bc2389cc7f0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -309,12 +309,13 @@ static void svm_range_free(struct svm_range *prange, bool
do_unmap)
}
static void
-svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc,
-uint8_t *granularity, uint32_t *flags)
+svm_range_set_default_attributes(struct svm_range_list *svms, int32_t
*location,
+int32_t *prefetch_loc, uint8_t *granularity,
+uint32_t *flags)
{
*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
- *granularity = 9;
+ *granularity = svms->default_granularity;
*flags =
KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;
}
@@ -358,9 +359,10 @@ svm_range *svm_range_new(struct svm_range_list *svms,
uint64_t start,
bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
MAX_GPU_INSTANCE);
- svm_range_set_default_attributes(&prange->preferred_loc,
+ svm_range_set_default_attributes(svms, &prange->preferred_loc,
&prange->prefetch_loc,
-&prange->granularity, &prange->flags);
+&prange->granularity,
+&prange->flags);
pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
@@ -2694,9 +2696,10 @@ svm_range_get_range_boundaries(struct kfd_process *p,
int64_t addr,
*is_h
[PATCH V4] drm/amdgpu: Surface svm_default_granularity, a RW module parameter
Enables users to update SVM's default granularity, used in
buffer migration and handling of recoverable page faults.
Param value is set in terms of log(numPages(buffer)),
e.g. 9 for a 2 MIB buffer
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 17 +
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 6 ++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 25 -
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e8c284aea1f2..8eb934af02f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -237,6 +237,7 @@ extern int sched_policy;
extern bool debug_evictions;
extern bool no_system_mem_limit;
extern int halt_if_hws_hang;
+extern uint amdgpu_svm_default_granularity;
#else
static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b9529948f2b2..442039436cb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -169,6 +169,16 @@ uint amdgpu_sdma_phase_quantum = 32;
char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
bool enforce_isolation;
+
+/* Specifies the default granularity for SVM, used in buffer
+ * migration and restoration of backing memory when handling
+ * recoverable page faults.
+ *
+ * The value is given as log(numPages(buffer)); for a 2 MiB
+ * buffer it computes to be 9
+ */
+uint amdgpu_svm_default_granularity = 9;
+
/*
* OverDrive(bit 14) disabled by default
* GFX DCS(bit 19) disabled by default
@@ -320,6 +330,13 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
+/**
+ * DOC: svm_default_granularity (uint)
+ * Used in buffer migration and handling of recoverable page faults
+ */
+MODULE_PARM_DESC(svm_default_granularity, "SVM's default granularity in
log(2^Pages), default 9 = 2^9 = 2 MiB");
+module_param_named(svm_default_granularity, amdgpu_svm_default_granularity,
uint, 0644);
+
/**
* DOC: lockup_timeout (string)
* Set GPU scheduler timeout value in ms.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9ae9abc6eb43..d6530febabad 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -868,6 +868,12 @@ struct svm_range_list {
struct task_struct *faulting_task;
/* check point ts decides if page fault recovery need be dropped */
uint64_tcheckpoint_ts[MAX_GPU_INSTANCE];
+
+ /* Default granularity to use in buffer migration
+* and restoration of backing memory while handling
+* recoverable page faults
+*/
+ uint8_t default_granularity;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index b44dec90969f..092cbb124319 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -309,12 +309,13 @@ static void svm_range_free(struct svm_range *prange, bool
do_unmap)
}
static void
-svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc,
-uint8_t *granularity, uint32_t *flags)
+svm_range_set_default_attributes(struct svm_range_list *svms, int32_t
*location,
+uint8_t *granularity, int32_t *prefetch_loc,
+uint32_t *flags)
{
*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
- *granularity = 9;
+ *granularity = svms->default_granularity;
*flags =
KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;
}
@@ -358,9 +359,10 @@ svm_range *svm_range_new(struct svm_range_list *svms,
uint64_t start,
bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
MAX_GPU_INSTANCE);
- svm_range_set_default_attributes(&prange->preferred_loc,
+ svm_range_set_default_attributes(svms, &prange->preferred_loc,
+&prange->granularity,
&prange->prefetch_loc,
-&prange->granularity, &prange->flags);
+&prange->flags);
pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
@@ -2694,9 +2696,10 @@ svm_range_get_range_boundaries(struct kfd_process *p,
int64_t addr,
*is_h
[PATCH V3] drm/amdgpu: Surface svm_default_granularity, a RW module parameter
Enables users to update SVM's default granularity, used in
buffer migration and handling of recoverable page faults.
Param value is set in terms of log(numPages(buffer)),
e.g. 9 for a 2 MIB buffer
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 17 +
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 6 ++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 25 +++--
4 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e8c284aea1f2..8eb934af02f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -237,6 +237,7 @@ extern int sched_policy;
extern bool debug_evictions;
extern bool no_system_mem_limit;
extern int halt_if_hws_hang;
+extern uint amdgpu_svm_default_granularity;
#else
static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b9529948f2b2..442039436cb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -169,6 +169,16 @@ uint amdgpu_sdma_phase_quantum = 32;
char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
bool enforce_isolation;
+
+/* Specifies the default granularity for SVM, used in buffer
+ * migration and restoration of backing memory when handling
+ * recoverable page faults.
+ *
+ * The value is given as log(numPages(buffer)); for a 2 MiB
+ * buffer it computes to be 9
+ */
+uint amdgpu_svm_default_granularity = 9;
+
/*
* OverDrive(bit 14) disabled by default
* GFX DCS(bit 19) disabled by default
@@ -320,6 +330,13 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
+/**
+ * DOC: svm_default_granularity (uint)
+ * Used in buffer migration and handling of recoverable page faults
+ */
+MODULE_PARM_DESC(svm_default_granularity, "SVM's default granularity in
log(2^Pages), default 9 = 2^9 = 2 MiB");
+module_param_named(svm_default_granularity, amdgpu_svm_default_granularity,
uint, 0644);
+
/**
* DOC: lockup_timeout (string)
* Set GPU scheduler timeout value in ms.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9ae9abc6eb43..d6530febabad 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -868,6 +868,12 @@ struct svm_range_list {
struct task_struct *faulting_task;
/* check point ts decides if page fault recovery need be dropped */
uint64_tcheckpoint_ts[MAX_GPU_INSTANCE];
+
+ /* Default granularity to use in buffer migration
+* and restoration of backing memory while handling
+* recoverable page faults
+*/
+ uint8_t default_granularity;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index b44dec90969f..624bfe317c9c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -309,12 +309,13 @@ static void svm_range_free(struct svm_range *prange, bool
do_unmap)
}
static void
-svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc,
-uint8_t *granularity, uint32_t *flags)
+svm_range_set_default_attributes(struct svm_range_list *svms,
+ int32_t *location, uint8_t *granularity,
+ int32_t *prefetch_loc, uint32_t *flags)
{
*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
- *granularity = 9;
+ *granularity = svms->default_granularity;
*flags =
KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;
}
@@ -358,9 +359,8 @@ svm_range *svm_range_new(struct svm_range_list *svms,
uint64_t start,
bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
MAX_GPU_INSTANCE);
- svm_range_set_default_attributes(&prange->preferred_loc,
-&prange->prefetch_loc,
-&prange->granularity, &prange->flags);
+ svm_range_set_default_attributes(svms, &prange->preferred_loc,
+ &prange->granularity, &prange->prefetch_loc, &prange->flags);
pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
@@ -2694,9 +2694,10 @@ svm_range_get_range_boundaries(struct kfd_process *p,
int64_t addr,
*is_heap_stack = vma_is_initial_heap(vma) || vma_is_initial_
[PATCH v2] drm/amdgpu: Surface svm_attr_gobm, a RW module parameter
Enables users to update the default size of buffer used
in migration either from Sysmem to VRAM or vice versa.
The param GOBM refers to granularity of buffer migration,
and is specified in terms of log(numPages(buffer)). It
facilitates users of unregistered memory to control GOBM,
albeit at a coarse level
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 18 +
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 12
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 26 -
4 files changed, 51 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e8c284aea1f2..73dd816b01f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -237,6 +237,7 @@ extern int sched_policy;
extern bool debug_evictions;
extern bool no_system_mem_limit;
extern int halt_if_hws_hang;
+extern uint amdgpu_svm_attr_gobm;
#else
static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
@@ -313,6 +314,9 @@ extern int amdgpu_wbrf;
/* Extra time delay(in ms) to eliminate the influence of temperature momentary
fluctuation */
#define AMDGPU_SWCTF_EXTRA_DELAY 50
+/* Default size of buffer to use in migrating buffer */
+#define AMDGPU_SVM_ATTR_GOBM 9
+
struct amdgpu_xcp_mgr;
struct amdgpu_device;
struct amdgpu_irq_src;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b9529948f2b2..09c501753a3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -169,6 +169,17 @@ uint amdgpu_sdma_phase_quantum = 32;
char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
bool enforce_isolation;
+
+/* Specifies the default size of buffer to use in
+ * migrating buffer from Sysmem to VRAM and vice
+ * versa
+ *
+ * GOBM - Granularity of Buffer Migration
+ *
+ * Defined as log2(sizeof(buffer)/PAGE_SIZE)
+ */
+uint amdgpu_svm_attr_gobm = AMDGPU_SVM_ATTR_GOBM;
+
/*
* OverDrive(bit 14) disabled by default
* GFX DCS(bit 19) disabled by default
@@ -320,6 +331,13 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
+/**
+ * DOC: svm_attr_gobm (uint)
+ * Size of buffer to use in migrating buffer from Sysmem to VRAM and vice versa
+ */
+MODULE_PARM_DESC(svm_attr_gobm, "Defined as log2(sizeof(buffer)/PAGE_SIZE),
e.g. 9 for 2 MiB");
+module_param_named(svm_attr_gobm, amdgpu_svm_attr_gobm, uint, 0644);
+
/**
* DOC: lockup_timeout (string)
* Set GPU scheduler timeout value in ms.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9ae9abc6eb43..c2e54b18c167 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -868,6 +868,18 @@ struct svm_range_list {
struct task_struct *faulting_task;
/* check point ts decides if page fault recovery need be dropped */
uint64_tcheckpoint_ts[MAX_GPU_INSTANCE];
+
+ /* Indicates the default size to use in migrating
+* buffers of a process from Sysmem to VRAM and vice
+* versa. The max legal value cannot be greater than
+* 0x3F
+*
+* @note: A side effect of this symbol being part of
+* struct svm_range_list is that it forces all buffers
+* of the process of unregistered kind to use the same
+* size in buffer migration
+*/
+ uint8_t attr_gobm;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index b44dec90969f..78c78baddb1f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -309,12 +309,11 @@ static void svm_range_free(struct svm_range *prange, bool
do_unmap)
}
static void
-svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc,
-uint8_t *granularity, uint32_t *flags)
+svm_range_set_default_attributes(int32_t *location,
+ int32_t *prefetch_loc, uint32_t *flags)
{
*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
- *granularity = 9;
*flags =
KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;
}
@@ -358,9 +357,9 @@ svm_range *svm_range_new(struct svm_range_list *svms,
uint64_t start,
bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
MAX_GPU_INSTANCE);
+ prange->granularity = svms->attr_gobm;
svm_range_set_default_a
[PATCH] drm/amdgpu: Surface svm_attr_gobm, a RW module parameter
KFD's design of unified memory (UM) does not allow users to
configure the size of buffer used in migrating buffer either
from Sysmem to VRAM or vice versa. This patch remedies this
gap, albeit at a coarse level, for workloads that deal with
unregistered memory
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 16 ++
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 12 ++
drivers/gpu/drm/amd/amdkfd/kfd_svm.c| 29 +
4 files changed, 52 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e8c284aea1f2..73dd816b01f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -237,6 +237,7 @@ extern int sched_policy;
extern bool debug_evictions;
extern bool no_system_mem_limit;
extern int halt_if_hws_hang;
+extern uint amdgpu_svm_attr_gobm;
#else
static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
@@ -313,6 +314,9 @@ extern int amdgpu_wbrf;
/* Extra time delay(in ms) to eliminate the influence of temperature momentary
fluctuation */
#define AMDGPU_SWCTF_EXTRA_DELAY 50
+/* Default size of buffer to use in migrating buffer */
+#define AMDGPU_SVM_ATTR_GOBM 9
+
struct amdgpu_xcp_mgr;
struct amdgpu_device;
struct amdgpu_irq_src;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b9529948f2b2..e195e1cf0f28 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -169,6 +169,15 @@ uint amdgpu_sdma_phase_quantum = 32;
char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
bool enforce_isolation;
+
+/* Specifies the default size of buffer to use in
+ * migrating buffer from Sysmem to VRAM and vice
+ * versa
+ *
+ * Defined as log2(sizeof(buffer)/PAGE_SIZE)
+ */
+uint amdgpu_svm_attr_gobm = AMDGPU_SVM_ATTR_GOBM;
+
/*
* OverDrive(bit 14) disabled by default
* GFX DCS(bit 19) disabled by default
@@ -320,6 +329,13 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
+/**
+ * DOC: svm_attr_gobm (uint)
+ * Size of buffer to use in migrating buffer from Sysmem to VRAM and vice versa
+ */
+MODULE_PARM_DESC(svm_attr_gobm, "Defined as log2(sizeof(buffer)/PAGE_SIZE),
e.g. 9 for 2 MiB");
+module_param_named(svm_attr_gobm, amdgpu_svm_attr_gobm, uint, 0644);
+
/**
* DOC: lockup_timeout (string)
* Set GPU scheduler timeout value in ms.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 7bba6bed2f48..07b202ab008a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -866,6 +866,18 @@ struct svm_range_list {
struct delayed_work restore_work;
DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
struct task_struct *faulting_task;
+
+ /* Indicates the default size to use in migrating
+* buffers of a process from Sysmem to VRAM and vice
+* versa. The max legal value cannot be greater than
+* 0x3F
+*
+* @note: A side effect of this symbol being part of
+* struct svm_range_list is that it forces all buffers
+* of the process of unregistered kind to use the same
+* size in buffer migration
+*/
+ uint8_t attr_gobm;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 10b1a1f64198..fcfe5543a3c0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -309,12 +309,11 @@ static void svm_range_free(struct svm_range *prange, bool
do_unmap)
}
static void
-svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc,
-uint8_t *granularity, uint32_t *flags)
+svm_range_set_default_attributes(int32_t *location,
+ int32_t *prefetch_loc, uint32_t *flags)
{
*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
- *granularity = 9;
*flags =
KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;
}
@@ -358,9 +357,9 @@ svm_range *svm_range_new(struct svm_range_list *svms,
uint64_t start,
bitmap_copy(prange->bitmap_access, svms->bitmap_supported,
MAX_GPU_INSTANCE);
+ prange->granularity = svms->attr_gobm;
svm_range_set_default_attributes(&prange->preferred_loc,
-&prange->prefetch_loc,
-
[PATCH] drm/amdgpu: Apply IH Ptr mask after updating IH Read Ptr
Apply IH Ptr mask immediately after updating to IH Read ptr. Ideally the operation to update and mask should be atomic. This will ensure that Read Ptr points to a valid index in the IH ring buffer. Signed-off-by: Ramesh Errabolu --- drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c | 5 - drivers/gpu/drm/amd/amdgpu/cik_ih.c | 5 - drivers/gpu/drm/amd/amdgpu/cz_ih.c | 5 - drivers/gpu/drm/amd/amdgpu/iceland_ih.c | 5 - drivers/gpu/drm/amd/amdgpu/si_ih.c | 4 drivers/gpu/drm/amd/amdgpu/tonga_ih.c | 5 - 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index f3b0aaf3ebc6..2fd13d424213 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -280,8 +280,11 @@ void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, entry->src_data[2] = dw[6]; entry->src_data[3] = dw[7]; - /* wptr/rptr are in bytes! */ + /* Apply IH Ptr mask after adding sizeof(Intrpt) +* bytes to IH Read ptr +*/ ih->rptr += 32; + ih->rptr &= ih->ptr_mask; } uint64_t amdgpu_ih_decode_iv_ts_helper(struct amdgpu_ih_ring *ih, u32 rptr, diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c index 576baa9dbb0e..41c0c56d41f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c @@ -265,8 +265,11 @@ static void cik_ih_decode_iv(struct amdgpu_device *adev, entry->vmid = (dw[2] >> 8) & 0xff; entry->pasid = (dw[2] >> 16) & 0x; - /* wptr/rptr are in bytes! */ + /* Apply IH Ptr mask after adding sizeof(Intrpt) +* bytes to IH Read ptr +*/ ih->rptr += 16; + ih->rptr &= ih->ptr_mask; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c index 072643787384..a94ddecc3d33 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c @@ -256,8 +256,11 @@ static void cz_ih_decode_iv(struct amdgpu_device *adev, entry->vmid = (dw[2] >> 8) & 0xff; entry->pasid = (dw[2] >> 16) & 0x; - /* wptr/rptr are in bytes! */ + /* Apply IH Ptr mask after adding sizeof(Intrpt) +* bytes to IH Read ptr +*/ ih->rptr += 16; + ih->rptr &= ih->ptr_mask; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c index 07984f7c3ae7..d47f008110ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c @@ -255,8 +255,11 @@ static void iceland_ih_decode_iv(struct amdgpu_device *adev, entry->vmid = (dw[2] >> 8) & 0xff; entry->pasid = (dw[2] >> 16) & 0x; - /* wptr/rptr are in bytes! */ + /* Apply IH Ptr mask after adding sizeof(Intrpt) +* bytes to IH Read ptr +*/ ih->rptr += 16; + ih->rptr &= ih->ptr_mask; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c index 5237395e4fab..9e51024f1851 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c @@ -147,7 +147,11 @@ static void si_ih_decode_iv(struct amdgpu_device *adev, entry->ring_id = dw[2] & 0xff; entry->vmid = (dw[2] >> 8) & 0xff; + /* Apply IH Ptr mask after adding sizeof(Intrpt) +* bytes to IH Read ptr +*/ ih->rptr += 16; + ih->rptr &= ih->ptr_mask; } static void si_ih_set_rptr(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c index 24d49d813607..db3c7aeb9425 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c @@ -259,8 +259,11 @@ static void tonga_ih_decode_iv(struct amdgpu_device *adev, entry->vmid = (dw[2] >> 8) & 0xff; entry->pasid = (dw[2] >> 16) & 0x; - /* wptr/rptr are in bytes! */ + /* Apply IH Ptr mask after adding sizeof(Intrpt) +* bytes to IH Read ptr +*/ ih->rptr += 16; + ih->rptr &= ih->ptr_mask; } /** -- 2.34.1
[PATCH v2] drm/amd/amdkfd: Fix a resource leak in svm_range_validate_and_map()
Analysis of code by Coverity, a static code analyser, has identified
a resource leak in the symbol hmm_range. This leak occurs when one of
the prior steps before it is released encounters an error.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 386875e6eb96..481cb958e165 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1658,7 +1658,7 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
start = map_start << PAGE_SHIFT;
end = (map_last + 1) << PAGE_SHIFT;
for (addr = start; !r && addr < end; ) {
- struct hmm_range *hmm_range;
+ struct hmm_range *hmm_range = NULL;
unsigned long map_start_vma;
unsigned long map_last_vma;
struct vm_area_struct *vma;
@@ -1696,7 +1696,12 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
}
svm_range_lock(prange);
- if (!r && amdgpu_hmm_range_get_pages_done(hmm_range)) {
+
+ /* Free backing memory of hmm_range if it was initialized
+* Overrride return value to TRY AGAIN only if prior returns
+* were successful
+*/
+ if (hmm_range && amdgpu_hmm_range_get_pages_done(hmm_range) &&
!r) {
pr_debug("hmm update the range, need validate again\n");
r = -EAGAIN;
}
--
2.34.1
[PATCH] drm/amd/amdkfd: Fix a resource leak in svm_range_validate_and_map()
Analysis of code by Coverity, a static code analyser, has identified
a resource leak in the symbol hmm_range. This leak occurs when one of
the prior steps before it is released encounters an error.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 386875e6eb96..dcb1d5d3f860 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1658,7 +1658,7 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
start = map_start << PAGE_SHIFT;
end = (map_last + 1) << PAGE_SHIFT;
for (addr = start; !r && addr < end; ) {
- struct hmm_range *hmm_range;
+ struct hmm_range *hmm_range = NULL;
unsigned long map_start_vma;
unsigned long map_last_vma;
struct vm_area_struct *vma;
@@ -1696,7 +1696,9 @@ static int svm_range_validate_and_map(struct mm_struct
*mm,
}
svm_range_lock(prange);
- if (!r && amdgpu_hmm_range_get_pages_done(hmm_range)) {
+
+ // Free backing memory of hmm_range if it was initialized
+ if (hmm_range && amdgpu_hmm_range_get_pages_done(hmm_range)) {
pr_debug("hmm update the range, need validate again\n");
r = -EAGAIN;
}
--
2.34.1
[PATCH] dma-buf: Correct the documentation of name and exp_name symbols
Fix the documentation of struct dma_buf members name and exp_name
as to how these members are to be used and accessed.
Signed-off-by: Ramesh Errabolu
---
include/linux/dma-buf.h | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 3f31baa3293f..8ff4add71f88 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -343,16 +343,19 @@ struct dma_buf {
/**
* @exp_name:
*
-* Name of the exporter; useful for debugging. See the
-* DMA_BUF_SET_NAME IOCTL.
+* Name of the exporter; useful for debugging. Must not be NULL
*/
const char *exp_name;
/**
* @name:
*
-* Userspace-provided name; useful for accounting and debugging,
-* protected by dma_resv_lock() on @resv and @name_lock for read access.
+* Userspace-provided name. Default value is NULL. If not NULL,
+* length cannot be longer than DMA_BUF_NAME_LEN, including NIL
+* char. Useful for accounting and debugging. Read/Write accesses
+* are protected by @name_lock
+*
+* See the IOCTLs DMA_BUF_SET_NAME or DMA_BUF_SET_NAME_A/B
*/
const char *name;
--
2.34.1
[PATCH] dma-buf: Correct the documentation of name and exp_name symbols
Fix the documentation of struct dma_buf members name and exp_name
as to how these members are to be used and accessed.
Signed-off-by: Ramesh Errabolu
---
include/linux/dma-buf.h | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 3f31baa3293f..8ff4add71f88 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -343,16 +343,19 @@ struct dma_buf {
/**
* @exp_name:
*
-* Name of the exporter; useful for debugging. See the
-* DMA_BUF_SET_NAME IOCTL.
+* Name of the exporter; useful for debugging. Must not be NULL
*/
const char *exp_name;
/**
* @name:
*
-* Userspace-provided name; useful for accounting and debugging,
-* protected by dma_resv_lock() on @resv and @name_lock for read access.
+* Userspace-provided name. Default value is NULL. If not NULL,
+* length cannot be longer than DMA_BUF_NAME_LEN, including NIL
+* char. Useful for accounting and debugging. Read/Write accesses
+* are protected by @name_lock
+*
+* See the IOCTLs DMA_BUF_SET_NAME or DMA_BUF_SET_NAME_A/B
*/
const char *name;
--
2.34.1
[PATCH] drm/amd: Enable checkpoint and restore of VRAM Bos with no VA
Tag VRAM BOs that do not have a VA with a unique Id, a 128-bit
UUID. This unique Id is used to distinguish BOs that might
otherwise be of same size. Checkpoint and restore assumes
that these BOs are not imported into a DRM device that is
accessible either from current process or its parent or
child process
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/Makefile | 3 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 3 +-
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 29 ++-
drivers/gpu/drm/amd/amdgpu/amdgpu_criu.c | 190 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_criu.h | 103 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h| 17 ++
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 30 ++-
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 3 +-
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 2 +-
9 files changed, 370 insertions(+), 10 deletions(-)
create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_criu.c
create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_criu.h
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
b/drivers/gpu/drm/amd/amdgpu/Makefile
index 260e32ef7bae..851e2c4db372 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -270,7 +270,8 @@ amdgpu-y += \
amdgpu_amdkfd_gc_9_4_3.o \
amdgpu_amdkfd_gfx_v10.o \
amdgpu_amdkfd_gfx_v10_3.o \
- amdgpu_amdkfd_gfx_v11.o
+ amdgpu_amdkfd_gfx_v11.o \
+ amdgpu_criu.o
ifneq ($(CONFIG_DRM_AMDGPU_CIK),)
amdgpu-y += amdgpu_amdkfd_gfx_v7.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index fcf8a98ad15e..6c0d7e6a66cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -289,7 +289,8 @@ size_t amdgpu_amdkfd_get_available_memory(struct
amdgpu_device *adev,
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
struct amdgpu_device *adev, uint64_t va, uint64_t size,
void *drm_priv, struct kgd_mem **mem,
- uint64_t *offset, uint32_t flags, bool criu_resume);
+ uint64_t *offset, uint32_t flags,
+ bool criu_resume, uuid_t *uuid);
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
uint64_t *size);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 2e302956a279..b139ffd519e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -25,6 +25,7 @@
#include
#include
#include
+#include
#include
#include "amdgpu_object.h"
@@ -35,6 +36,7 @@
#include "amdgpu_dma_buf.h"
#include
#include "amdgpu_xgmi.h"
+#include "amdgpu_criu.h"
#include "kfd_priv.h"
#include "kfd_smi_events.h"
@@ -1718,7 +1720,8 @@ size_t amdgpu_amdkfd_get_available_memory(struct
amdgpu_device *adev,
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
struct amdgpu_device *adev, uint64_t va, uint64_t size,
void *drm_priv, struct kgd_mem **mem,
- uint64_t *offset, uint32_t flags, bool criu_resume)
+ uint64_t *offset, uint32_t flags,
+ bool criu_resume, uuid_t *uuid)
{
struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
@@ -1814,13 +1817,23 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
va, (*mem)->aql_queue ? size << 1 : size,
domain_string(alloc_domain), xcp_id);
- ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain,
alloc_flags,
- bo_type, NULL, &gobj, xcp_id + 1);
+ /* Construction of VRAM BO one with no VA, during CRIU Restore
+* should consult BO table. Will return either a previously
+* constructed BO or will construct a BO anew
+*/
+ if (criu_resume && (va == 0) && (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM))
+ ret = restore_vram_bo(adev, aligned_size, 1, alloc_flags, uuid,
+ &gobj, xcp_id + 1);
+ else
+ ret = amdgpu_gem_object_create(adev, aligned_size, 1,
alloc_domain,
+ alloc_flags, bo_type, NULL, &gobj, xcp_id + 1);
+
if (ret) {
pr_debug("Failed to create BO on domain %s. ret %d\n",
domain_string(alloc_domain), ret);
goto err_bo_create;
}
+
ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);
if (ret) {
pr_debug("Failed to allow vma node access. ret %d\n", ret);
@@ -1843,6 +1856,16 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gp
[Patch V2 v2] drm/amdgpu: Checkpoint and Restore VRAM BOs without VA
Extend checkpoint logic to allow inclusion of VRAM BOs that
do not have a VA attached
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 40ac093b5035..44c647c82070 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1845,7 +1845,8 @@ static uint32_t get_process_num_bos(struct kfd_process *p)
idr_for_each_entry(&pdd->alloc_idr, mem, id) {
struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
- if ((uint64_t)kgd_mem->va > pdd->gpuvm_base)
+ if (((uint64_t)kgd_mem->va > pdd->gpuvm_base) ||
+ !kgd_mem->va)
num_of_bos++;
}
}
@@ -1917,7 +1918,11 @@ static int criu_checkpoint_bos(struct kfd_process *p,
kgd_mem = (struct kgd_mem *)mem;
dumper_bo = kgd_mem->bo;
- if ((uint64_t)kgd_mem->va <= pdd->gpuvm_base)
+ /* Skip checkpointing BOs that are used for Trap handler
+* code and state. Currently, these BOs have a VA that
+* is less GPUVM Base
+*/
+ if (((uint64_t)kgd_mem->va <= pdd->gpuvm_base) &&
kgd_mem->va)
continue;
bo_bucket = &bo_buckets[bo_index];
--
2.25.1
[PATCH] drm/amdgpu: Checkpoint and Restore VRAM BOs without VA
Extend checkpoint logic to allow inclusion of VRAM BOs that
do not have a VA attached
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 40ac093b5035..5cc00ff4b635 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1845,7 +1845,8 @@ static uint32_t get_process_num_bos(struct kfd_process *p)
idr_for_each_entry(&pdd->alloc_idr, mem, id) {
struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
- if ((uint64_t)kgd_mem->va > pdd->gpuvm_base)
+ if (((uint64_t)kgd_mem->va > pdd->gpuvm_base) ||
+ (kgd_mem->va == 0))
num_of_bos++;
}
}
@@ -1917,7 +1918,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
kgd_mem = (struct kgd_mem *)mem;
dumper_bo = kgd_mem->bo;
- if ((uint64_t)kgd_mem->va <= pdd->gpuvm_base)
+ if (((uint64_t)kgd_mem->va <= pdd->gpuvm_base) &&
+ !(kgd_mem->va == 0))
continue;
bo_bucket = &bo_buckets[bo_index];
--
2.25.1
[patch V2] drm/amdkfd: Access gpuvm_export_dmabuf() api
Call KFD api to get Dmabuf instead of calling GEM Prime API
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 11 +--
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index cf1db0ab3471..40ac093b5035 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1852,15 +1852,14 @@ static uint32_t get_process_num_bos(struct kfd_process
*p)
return num_of_bos;
}
-static int criu_get_prime_handle(struct drm_gem_object *gobj, int flags,
+static int criu_get_prime_handle(struct kgd_mem *mem, int flags,
u32 *shared_fd)
{
struct dma_buf *dmabuf;
int ret;
- dmabuf = amdgpu_gem_prime_export(gobj, flags);
- if (IS_ERR(dmabuf)) {
- ret = PTR_ERR(dmabuf);
+ ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
+ if (ret) {
pr_err("dmabuf export failed for the BO\n");
return ret;
}
@@ -1940,7 +1939,7 @@ static int criu_checkpoint_bos(struct kfd_process *p,
}
if (bo_bucket->alloc_flags
& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM |
KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
- ret =
criu_get_prime_handle(&dumper_bo->tbo.base,
+ ret = criu_get_prime_handle(kgd_mem,
bo_bucket->alloc_flags &
KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
&bo_bucket->dmabuf_fd);
@@ -2402,7 +2401,7 @@ static int criu_restore_bo(struct kfd_process *p,
/* create the dmabuf object and export the bo */
if (bo_bucket->alloc_flags
& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
- ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR,
+ ret = criu_get_prime_handle(kgd_mem, DRM_RDWR,
&bo_bucket->dmabuf_fd);
if (ret)
return ret;
--
2.25.1
[PATCH] drm/amdkfd: Access gpuvm_export_dmabuf() api
Call KFD api to get Dmabuf instead of calling GEM Prime API
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index cf1db0ab3471..c37d82b35372 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1852,13 +1852,13 @@ static uint32_t get_process_num_bos(struct kfd_process
*p)
return num_of_bos;
}
-static int criu_get_prime_handle(struct drm_gem_object *gobj, int flags,
+static int criu_get_prime_handle(struct kgd_mem *mem, int flags,
u32 *shared_fd)
{
struct dma_buf *dmabuf;
int ret;
- dmabuf = amdgpu_gem_prime_export(gobj, flags);
+ ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
if (IS_ERR(dmabuf)) {
ret = PTR_ERR(dmabuf);
pr_err("dmabuf export failed for the BO\n");
@@ -1940,7 +1940,7 @@ static int criu_checkpoint_bos(struct kfd_process *p,
}
if (bo_bucket->alloc_flags
& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM |
KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
- ret =
criu_get_prime_handle(&dumper_bo->tbo.base,
+ ret = criu_get_prime_handle(kgd_mem,
bo_bucket->alloc_flags &
KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
&bo_bucket->dmabuf_fd);
@@ -2402,7 +2402,7 @@ static int criu_restore_bo(struct kfd_process *p,
/* create the dmabuf object and export the bo */
if (bo_bucket->alloc_flags
& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
- ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR,
+ ret = criu_get_prime_handle(kgd_mem, DRM_RDWR,
&bo_bucket->dmabuf_fd);
if (ret)
return ret;
--
2.25.1
[PATCH] drm/amdgpu: Enable Aldebaran devices to report CU Occupancy
Allow user to know number of compute units (CU) that are in use at any
given moment. Enable access to the method kgd_gfx_v9_get_cu_occupancy
that computes CU occupancy.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
index c8935d718207..4485bb29bec9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
@@ -41,5 +41,6 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = {
.get_atc_vmid_pasid_mapping_info =
kgd_gfx_v9_get_atc_vmid_pasid_mapping_info,
.set_vm_context_page_table_base =
kgd_gfx_v9_set_vm_context_page_table_base,
+ .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy,
.program_trap_handler_settings =
kgd_gfx_v9_program_trap_handler_settings
};
--
2.35.1
[PATCH] x86/configs: Update defconfig with peer-to-peer configs
- Update defconfig for PCI_P2PDMA - Update defconfig for DMABUF_MOVE_NOTIFY - Update defconfig for HSA_AMD_P2P --- Notes: Following procedure was applied: make rock-dbg_defconfig make menuconfig Enable PCI_P2PDMA Enable DMABUF_MOVE_NOTIFY Enable HSA_AMD_P2P make savedefconfig cp defconfig rock-dbg_defconfig commit changes arch/x86/configs/rock-dbg_defconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/configs/rock-dbg_defconfig b/arch/x86/configs/rock-dbg_defconfig index 406fdfaceb55..0ad80a8c8eab 100644 --- a/arch/x86/configs/rock-dbg_defconfig +++ b/arch/x86/configs/rock-dbg_defconfig @@ -303,6 +303,7 @@ CONFIG_PCIEAER=y CONFIG_PCI_REALLOC_ENABLE_AUTO=y CONFIG_PCI_STUB=y CONFIG_PCI_IOV=y +CONFIG_PCI_P2PDMA=y CONFIG_HOTPLUG_PCI=y CONFIG_RAPIDIO=y CONFIG_RAPIDIO_DMA_ENGINE=y @@ -417,6 +418,7 @@ CONFIG_DRM_AMDGPU=m CONFIG_DRM_AMDGPU_SI=y CONFIG_DRM_AMDGPU_CIK=y CONFIG_HSA_AMD=y +CONFIG_HSA_AMD_P2P=y CONFIG_DRM_AST=m CONFIG_FB=y CONFIG_BACKLIGHT_CLASS_DEVICE=y @@ -453,6 +455,7 @@ CONFIG_LEDS_TRIGGERS=y CONFIG_RTC_CLASS=y # CONFIG_RTC_HCTOSYS is not set CONFIG_DMADEVICES=y +CONFIG_DMABUF_MOVE_NOTIFY=y # CONFIG_X86_PLATFORM_DEVICES is not set CONFIG_AMD_IOMMU=y CONFIG_INTEL_IOMMU=y -- 2.35.1
[PATCH] drm/amdkfd: Fix warnings from static analyzer Smatch
The patch fixes warnings/error as reported by Smatch a static analyzer
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1428
kfd_create_indirect_link_prop() warn: iterator used outside loop: 'cpu_link'
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1462
kfd_create_indirect_link_prop() error: we previously assumed 'cpu_dev' could be
null (see line 1420)
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1516 kfd_add_peer_prop()
warn: iterator used outside loop: 'iolink3'
Fixes: 40d6aa758b13 ("drm/amdkfd: Extend KFD device topology to surface
peer-to-peer links")
Signed-off-by: Ramesh Errabolu
Reported-by: Dan Carpenter
Reviewed-by: Felix Kuehling
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 34 +++
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 25990bec600d..ca4825e555b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1417,15 +1417,15 @@ static int kfd_create_indirect_link_prop(struct
kfd_topology_device *kdev, int g
/* find CPU <--> CPU links */
cpu_dev = kfd_topology_device_by_proximity_domain(i);
- if (cpu_dev) {
- list_for_each_entry(cpu_link,
- &cpu_dev->io_link_props, list) {
- if (cpu_link->node_to == gpu_link->node_to)
- break;
- }
- }
+ if (!cpu_dev)
+ continue;
+
+ list_for_each_entry(cpu_link, &cpu_dev->io_link_props, list)
+ if (cpu_link->node_to == gpu_link->node_to)
+ break;
- if (cpu_link->node_to != gpu_link->node_to)
+ /* Ensures we didn't exit from list search with no hits */
+ if (list_entry_is_head(cpu_link, &cpu_dev->io_link_props, list))
return -ENOMEM;
/* CPU <--> CPU <--> GPU, GPU node*/
@@ -1510,16 +1510,16 @@ static int kfd_add_peer_prop(struct kfd_topology_device
*kdev,
cpu_dev =
kfd_topology_device_by_proximity_domain(iolink1->node_to);
if (cpu_dev) {
list_for_each_entry(iolink3, &cpu_dev->io_link_props,
list)
- if (iolink3->node_to == iolink2->node_to)
+ if (iolink3->node_to == iolink2->node_to) {
+ props->weight += iolink3->weight;
+ props->min_latency +=
iolink3->min_latency;
+ props->max_latency +=
iolink3->max_latency;
+ props->min_bandwidth =
min(props->min_bandwidth,
+
iolink3->min_bandwidth);
+ props->max_bandwidth =
min(props->max_bandwidth,
+
iolink3->max_bandwidth);
break;
-
- props->weight += iolink3->weight;
- props->min_latency += iolink3->min_latency;
- props->max_latency += iolink3->max_latency;
- props->min_bandwidth = min(props->min_bandwidth,
- iolink3->min_bandwidth);
- props->max_bandwidth = min(props->max_bandwidth,
- iolink3->max_bandwidth);
+ }
} else {
WARN(1, "CPU node not found");
}
--
2.35.1
[PATCH] drm/amdkfd: Fix warnings from static analyzer Smatch
The patch fixes warnings/error as reported by Smatch a static analyzer
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1428
kfd_create_indirect_link_prop() warn: iterator used outside loop: 'cpu_link'
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1462
kfd_create_indirect_link_prop() error: we previously assumed 'cpu_dev' could be
null (see line 1420)
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1516 kfd_add_peer_prop()
warn: iterator used outside loop: 'iolink3'
Fixes: 40d6aa758b13 ("drm/amdkfd: Extend KFD device topology to surface
peer-to-peer links")
Signed-off-by: Ramesh Errabolu
Reported-by: Dan Carpenter
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 34 +++
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 25990bec600d..ca4825e555b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1417,15 +1417,15 @@ static int kfd_create_indirect_link_prop(struct
kfd_topology_device *kdev, int g
/* find CPU <--> CPU links */
cpu_dev = kfd_topology_device_by_proximity_domain(i);
- if (cpu_dev) {
- list_for_each_entry(cpu_link,
- &cpu_dev->io_link_props, list) {
- if (cpu_link->node_to == gpu_link->node_to)
- break;
- }
- }
+ if (!cpu_dev)
+ continue;
+
+ list_for_each_entry(cpu_link, &cpu_dev->io_link_props, list)
+ if (cpu_link->node_to == gpu_link->node_to)
+ break;
- if (cpu_link->node_to != gpu_link->node_to)
+ /* Ensures we didn't exit from list search with no hits */
+ if (list_entry_is_head(cpu_link, &cpu_dev->io_link_props, list))
return -ENOMEM;
/* CPU <--> CPU <--> GPU, GPU node*/
@@ -1510,16 +1510,16 @@ static int kfd_add_peer_prop(struct kfd_topology_device
*kdev,
cpu_dev =
kfd_topology_device_by_proximity_domain(iolink1->node_to);
if (cpu_dev) {
list_for_each_entry(iolink3, &cpu_dev->io_link_props,
list)
- if (iolink3->node_to == iolink2->node_to)
+ if (iolink3->node_to == iolink2->node_to) {
+ props->weight += iolink3->weight;
+ props->min_latency +=
iolink3->min_latency;
+ props->max_latency +=
iolink3->max_latency;
+ props->min_bandwidth =
min(props->min_bandwidth,
+
iolink3->min_bandwidth);
+ props->max_bandwidth =
min(props->max_bandwidth,
+
iolink3->max_bandwidth);
break;
-
- props->weight += iolink3->weight;
- props->min_latency += iolink3->min_latency;
- props->max_latency += iolink3->max_latency;
- props->min_bandwidth = min(props->min_bandwidth,
- iolink3->min_bandwidth);
- props->max_bandwidth = min(props->max_bandwidth,
- iolink3->max_bandwidth);
+ }
} else {
WARN(1, "CPU node not found");
}
--
2.35.1
[PATCH] drm/amdkfd: Fix warnings from static analyzer Smatch
The patch fixes couple of warnings, as reported by Smatch
a static analyzer.
Fixes: 40d6aa758b13 ("drm/amdkfd: Extend KFD device topology to surface
peer-to-peer links")
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1428
kfd_create_indirect_link_prop() warn: iterator used outside loop: 'cpu_link'
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1462
kfd_create_indirect_link_prop() error: we previously assumed 'cpu_dev' could be
null (see line 1420)
drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_topology.c:1516 kfd_add_peer_prop()
warn: iterator used outside loop: 'iolink3'
Signed-off-by: Ramesh Errabolu
Reported-by: Dan Carpenter
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 34 +++
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 25990bec600d..ca4825e555b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1417,15 +1417,15 @@ static int kfd_create_indirect_link_prop(struct
kfd_topology_device *kdev, int g
/* find CPU <--> CPU links */
cpu_dev = kfd_topology_device_by_proximity_domain(i);
- if (cpu_dev) {
- list_for_each_entry(cpu_link,
- &cpu_dev->io_link_props, list) {
- if (cpu_link->node_to == gpu_link->node_to)
- break;
- }
- }
+ if (!cpu_dev)
+ continue;
+
+ list_for_each_entry(cpu_link, &cpu_dev->io_link_props, list)
+ if (cpu_link->node_to == gpu_link->node_to)
+ break;
- if (cpu_link->node_to != gpu_link->node_to)
+ /* Ensures we didn't exit from list search with no hits */
+ if (list_entry_is_head(cpu_link, &cpu_dev->io_link_props, list))
return -ENOMEM;
/* CPU <--> CPU <--> GPU, GPU node*/
@@ -1510,16 +1510,16 @@ static int kfd_add_peer_prop(struct kfd_topology_device
*kdev,
cpu_dev =
kfd_topology_device_by_proximity_domain(iolink1->node_to);
if (cpu_dev) {
list_for_each_entry(iolink3, &cpu_dev->io_link_props,
list)
- if (iolink3->node_to == iolink2->node_to)
+ if (iolink3->node_to == iolink2->node_to) {
+ props->weight += iolink3->weight;
+ props->min_latency +=
iolink3->min_latency;
+ props->max_latency +=
iolink3->max_latency;
+ props->min_bandwidth =
min(props->min_bandwidth,
+
iolink3->min_bandwidth);
+ props->max_bandwidth =
min(props->max_bandwidth,
+
iolink3->max_bandwidth);
break;
-
- props->weight += iolink3->weight;
- props->min_latency += iolink3->min_latency;
- props->max_latency += iolink3->max_latency;
- props->min_bandwidth = min(props->min_bandwidth,
- iolink3->min_bandwidth);
- props->max_bandwidth = min(props->max_bandwidth,
- iolink3->max_bandwidth);
+ }
} else {
WARN(1, "CPU node not found");
}
--
2.35.1
[PATCH] drm/amdkfd: Fix warnings from static analyzer Smatch
The patch fixes couple of warnings, as reported by Smatch
a static analyzer
Signed-off-by: Ramesh Errabolu
Reported-by: Dan Carpenter
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 34 +++
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 25990bec600d..ca4825e555b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1417,15 +1417,15 @@ static int kfd_create_indirect_link_prop(struct
kfd_topology_device *kdev, int g
/* find CPU <--> CPU links */
cpu_dev = kfd_topology_device_by_proximity_domain(i);
- if (cpu_dev) {
- list_for_each_entry(cpu_link,
- &cpu_dev->io_link_props, list) {
- if (cpu_link->node_to == gpu_link->node_to)
- break;
- }
- }
+ if (!cpu_dev)
+ continue;
+
+ list_for_each_entry(cpu_link, &cpu_dev->io_link_props, list)
+ if (cpu_link->node_to == gpu_link->node_to)
+ break;
- if (cpu_link->node_to != gpu_link->node_to)
+ /* Ensures we didn't exit from list search with no hits */
+ if (list_entry_is_head(cpu_link, &cpu_dev->io_link_props, list))
return -ENOMEM;
/* CPU <--> CPU <--> GPU, GPU node*/
@@ -1510,16 +1510,16 @@ static int kfd_add_peer_prop(struct kfd_topology_device
*kdev,
cpu_dev =
kfd_topology_device_by_proximity_domain(iolink1->node_to);
if (cpu_dev) {
list_for_each_entry(iolink3, &cpu_dev->io_link_props,
list)
- if (iolink3->node_to == iolink2->node_to)
+ if (iolink3->node_to == iolink2->node_to) {
+ props->weight += iolink3->weight;
+ props->min_latency +=
iolink3->min_latency;
+ props->max_latency +=
iolink3->max_latency;
+ props->min_bandwidth =
min(props->min_bandwidth,
+
iolink3->min_bandwidth);
+ props->max_bandwidth =
min(props->max_bandwidth,
+
iolink3->max_bandwidth);
break;
-
- props->weight += iolink3->weight;
- props->min_latency += iolink3->min_latency;
- props->max_latency += iolink3->max_latency;
- props->min_bandwidth = min(props->min_bandwidth,
- iolink3->min_bandwidth);
- props->max_bandwidth = min(props->max_bandwidth,
- iolink3->max_bandwidth);
+ }
} else {
WARN(1, "CPU node not found");
}
--
2.35.1
[PATCH] drm/amdkfd: Fix warnings from static analyzer Smatch
The patch fixes couple of warnings, as reported by Smatch
a static analyzer
Signed-off-by: Ramesh Errabolu
Reported-by: Dan Carpenter
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 36 ---
1 file changed, 19 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 25990bec600d..9d7b9ad70bc8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1417,15 +1417,17 @@ static int kfd_create_indirect_link_prop(struct
kfd_topology_device *kdev, int g
/* find CPU <--> CPU links */
cpu_dev = kfd_topology_device_by_proximity_domain(i);
- if (cpu_dev) {
- list_for_each_entry(cpu_link,
- &cpu_dev->io_link_props, list) {
- if (cpu_link->node_to == gpu_link->node_to)
- break;
- }
- }
+ if (!cpu_dev)
+ continue;
+
+ cpu_link = NULL;
+ list_for_each_entry(cpu_link, &cpu_dev->io_link_props, list)
+ if (cpu_link->node_to == gpu_link->node_to)
+ break;
- if (cpu_link->node_to != gpu_link->node_to)
+ /* Ensures we didn't exit from list search with no hits */
+ if (list_entry_is_head(cpu_link, &cpu_dev->io_link_props, list)
||
+ (cpu_link->node_to != gpu_link->node_to))
return -ENOMEM;
/* CPU <--> CPU <--> GPU, GPU node*/
@@ -1510,16 +1512,16 @@ static int kfd_add_peer_prop(struct kfd_topology_device
*kdev,
cpu_dev =
kfd_topology_device_by_proximity_domain(iolink1->node_to);
if (cpu_dev) {
list_for_each_entry(iolink3, &cpu_dev->io_link_props,
list)
- if (iolink3->node_to == iolink2->node_to)
+ if (iolink3->node_to == iolink2->node_to) {
+ props->weight += iolink3->weight;
+ props->min_latency +=
iolink3->min_latency;
+ props->max_latency +=
iolink3->max_latency;
+ props->min_bandwidth =
min(props->min_bandwidth,
+
iolink3->min_bandwidth);
+ props->max_bandwidth =
min(props->max_bandwidth,
+
iolink3->max_bandwidth);
break;
-
- props->weight += iolink3->weight;
- props->min_latency += iolink3->min_latency;
- props->max_latency += iolink3->max_latency;
- props->min_bandwidth = min(props->min_bandwidth,
- iolink3->min_bandwidth);
- props->max_bandwidth = min(props->max_bandwidth,
- iolink3->max_bandwidth);
+ }
} else {
WARN(1, "CPU node not found");
}
--
2.35.1
[PATCH] drm/amdgpu: Fix error handling in amdgpu_amdkfd_gpuvm_free_memory_of_gpu
Following error conditions are fixed:
Unpin MMIO and DOORBELL BOs only after map count goes to zero
Remove BO from validate list of a KFD process in a safe manner
Print a warning message if unreserving GPUVMs encounters an error
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 42 +++
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index a1de900ba677..ee48e6591f99 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1013,14 +1013,22 @@ static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem
*mem,
mutex_unlock(&process_info->lock);
}
-static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
+/**
+ * remove_kgd_mem_from_validate_list() - Remove BO from process's validate
list,
+ * in an idempotent manner, so that restore worker can't access it anymore
+ * @mem: BO's membership handle in validate list
+ * @process_info: KFD process handle to which BO belongs
+ *
+ * Return: void
+ */
+static void remove_kgd_mem_from_validate_list(struct kgd_mem *mem,
struct amdkfd_process_info *process_info)
{
struct ttm_validate_buffer *bo_list_entry;
bo_list_entry = &mem->validate_list;
mutex_lock(&process_info->lock);
- list_del(&bo_list_entry->head);
+ list_del_init(&bo_list_entry->head);
mutex_unlock(&process_info->lock);
}
@@ -1796,7 +1804,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
allocate_init_user_pages_failed:
err_pin_bo:
- remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+ remove_kgd_mem_from_validate_list(*mem, avm->process_info);
drm_vma_node_revoke(&gobj->vma_node, drm_priv);
err_node_allow:
/* Don't unreserve system mem limit twice */
@@ -1825,20 +1833,12 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
unsigned long bo_size = mem->bo->tbo.base.size;
struct kfd_mem_attachment *entry, *tmp;
struct bo_vm_reservation_context ctx;
- struct ttm_validate_buffer *bo_list_entry;
unsigned int mapped_to_gpu_memory;
int ret;
bool is_imported = false;
mutex_lock(&mem->lock);
- /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
- if (mem->alloc_flags &
- (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
-KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
- amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
- }
-
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
@@ -1853,10 +1853,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
}
/* Make sure restore workers don't access the BO any more */
- bo_list_entry = &mem->validate_list;
- mutex_lock(&process_info->lock);
- list_del(&bo_list_entry->head);
- mutex_unlock(&process_info->lock);
+ remove_kgd_mem_from_validate_list(mem, process_info);
/* No more MMU notifiers */
amdgpu_mn_unregister(mem->bo);
@@ -1878,7 +1875,18 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
list_for_each_entry_safe(entry, tmp, &mem->attachments, list)
kfd_mem_detach(entry);
+ /* Return success even in case of error */
ret = unreserve_bo_and_vms(&ctx, false, false);
+ if (unlikely(ret)) {
+ WARN_ONCE(ret, "Error in unreserving BO and associated VMs");
+ ret = 0;
+ }
+
+ /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
/* Free the sync object */
amdgpu_sync_free(&mem->sync);
@@ -2814,7 +2822,7 @@ int amdgpu_amdkfd_add_gws_to_process(void *info, void
*gws, struct kgd_mem **mem
bo_reservation_failure:
mutex_unlock(&(*mem)->process_info->lock);
amdgpu_sync_free(&(*mem)->sync);
- remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
+ remove_kgd_mem_from_validate_list(*mem, process_info);
amdgpu_bo_unref(&gws_bo);
mutex_destroy(&(*mem)->lock);
kfree(*mem);
@@ -2832,7 +2840,7 @@ int amdgpu_amdkfd_remove_gws_from_process(void *info,
void *mem)
/* Remove BO from process's validate list so restore worker won't touch
* it anymore
*/
- remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
+ remove_kgd_mem_from_validate_list(kgd_mem, process_info);
ret = amdgpu_bo_reserve(gws_bo, false);
if (unlikely(ret)) {
--
2.35.1
[PATCH] drm/amdgpu: Fix error handling in amdgpu_amdkfd_gpuvm_free_memory_of_gpu
Following error conditions are fixed:
Ensure calls to delete a list element are safe
Unpin MMIO and DOORBELL BOs only after map count goes to zero
Print a warning message if unreserving VMs encounters an error
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 38 ++-
1 file changed, 20 insertions(+), 18 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index a1de900ba677..78b3efecc2f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1013,14 +1013,14 @@ static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem
*mem,
mutex_unlock(&process_info->lock);
}
-static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
+static void remove_kgd_mem_from_validate_list(struct kgd_mem *mem,
struct amdkfd_process_info *process_info)
{
struct ttm_validate_buffer *bo_list_entry;
bo_list_entry = &mem->validate_list;
mutex_lock(&process_info->lock);
- list_del(&bo_list_entry->head);
+ list_del_init(&bo_list_entry->head);
mutex_unlock(&process_info->lock);
}
@@ -1796,7 +1796,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
allocate_init_user_pages_failed:
err_pin_bo:
- remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+ remove_kgd_mem_from_validate_list(*mem, avm->process_info);
drm_vma_node_revoke(&gobj->vma_node, drm_priv);
err_node_allow:
/* Don't unreserve system mem limit twice */
@@ -1825,20 +1825,12 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
unsigned long bo_size = mem->bo->tbo.base.size;
struct kfd_mem_attachment *entry, *tmp;
struct bo_vm_reservation_context ctx;
- struct ttm_validate_buffer *bo_list_entry;
unsigned int mapped_to_gpu_memory;
int ret;
bool is_imported = false;
mutex_lock(&mem->lock);
- /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
- if (mem->alloc_flags &
- (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
-KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
- amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
- }
-
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
@@ -1852,11 +1844,10 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
return -EBUSY;
}
- /* Make sure restore workers don't access the BO any more */
- bo_list_entry = &mem->validate_list;
- mutex_lock(&process_info->lock);
- list_del(&bo_list_entry->head);
- mutex_unlock(&process_info->lock);
+ /* Make sure restore workers don't access the BO any more
+* Ensure removal of BO from validate list is reentrant safe
+*/
+ remove_kgd_mem_from_validate_list(mem, process_info);
/* No more MMU notifiers */
amdgpu_mn_unregister(mem->bo);
@@ -1878,7 +1869,18 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
list_for_each_entry_safe(entry, tmp, &mem->attachments, list)
kfd_mem_detach(entry);
+ /* Return success even in case of error */
ret = unreserve_bo_and_vms(&ctx, false, false);
+ if (unlikely(ret)) {
+ WARN_ONCE(ret, "Error in unreserving BO and associated VMs");
+ ret = 0;
+ }
+
+ /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
/* Free the sync object */
amdgpu_sync_free(&mem->sync);
@@ -2814,7 +2816,7 @@ int amdgpu_amdkfd_add_gws_to_process(void *info, void
*gws, struct kgd_mem **mem
bo_reservation_failure:
mutex_unlock(&(*mem)->process_info->lock);
amdgpu_sync_free(&(*mem)->sync);
- remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
+ remove_kgd_mem_from_validate_list(*mem, process_info);
amdgpu_bo_unref(&gws_bo);
mutex_destroy(&(*mem)->lock);
kfree(*mem);
@@ -2832,7 +2834,7 @@ int amdgpu_amdkfd_remove_gws_from_process(void *info,
void *mem)
/* Remove BO from process's validate list so restore worker won't touch
* it anymore
*/
- remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
+ remove_kgd_mem_from_validate_list(kgd_mem, process_info);
ret = amdgpu_bo_reserve(gws_bo, false);
if (unlikely(ret)) {
--
2.35.1
[PATCH] drm/amdkfd: Remove field io_link_count from struct kfd_topology_device
The field is redundant and does not serve any functional role
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 --
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 1 -
drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 1 -
3 files changed, 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index cbfb32b3d235..a5409531a2fd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1040,7 +1040,6 @@ static int kfd_parse_subtype_iolink(struct
crat_subtype_iolink *iolink,
props->rec_transfer_size =
iolink->recommended_transfer_size;
- dev->io_link_count++;
dev->node_props.io_links_count++;
list_add_tail(&props->list, &dev->io_link_props);
break;
@@ -1067,7 +1066,6 @@ static int kfd_parse_subtype_iolink(struct
crat_subtype_iolink *iolink,
props2->node_from = id_to;
props2->node_to = id_from;
props2->kobj = NULL;
- to_dev->io_link_count++;
to_dev->node_props.io_links_count++;
list_add_tail(&props2->list, &to_dev->io_link_props);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 3e240b22ec91..304322ac39e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1819,7 +1819,6 @@ static void kfd_topology_update_io_links(int
proximity_domain)
*/
if (iolink->node_to == proximity_domain) {
list_del(&iolink->list);
- dev->io_link_count--;
dev->node_props.io_links_count--;
} else {
if (iolink->node_from > proximity_domain)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index 2fb5664e1041..9f6c949186c1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -130,7 +130,6 @@ struct kfd_topology_device {
struct list_headmem_props;
uint32_tcache_count;
struct list_headcache_props;
- uint32_tio_link_count;
struct list_headio_link_props;
struct list_headp2p_link_props;
struct list_headperf_props;
--
2.35.1
[PATCH] drm/amdgpu: Unpin MMIO and DOORBELL BOs only after map count goes to zero
In existing code MMIO and DOORBELL BOs are unpinned without ensuring the
condition that their map count has reached zero. Unpinning without checking
this constraint could lead to an error while BO is being freed. The patch
fixes this issue.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 15 +++
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index a1de900ba677..e5dc94b745b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1832,13 +1832,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
mutex_lock(&mem->lock);
- /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
- if (mem->alloc_flags &
- (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
-KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
- amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
- }
-
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
@@ -1855,7 +1848,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
/* Make sure restore workers don't access the BO any more */
bo_list_entry = &mem->validate_list;
mutex_lock(&process_info->lock);
- list_del(&bo_list_entry->head);
+ list_del_init(&bo_list_entry->head);
mutex_unlock(&process_info->lock);
/* No more MMU notifiers */
@@ -1880,6 +1873,12 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
ret = unreserve_bo_and_vms(&ctx, false, false);
+ /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
+
/* Free the sync object */
amdgpu_sync_free(&mem->sync);
--
2.35.1
[PATCH] drm/amdkfd: Remove field io_link_count from struct kfd_topology_device
The field is redundant and does not serve any functional role
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 --
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 1 -
drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 1 -
3 files changed, 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index cbfb32b3d235..a5409531a2fd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1040,7 +1040,6 @@ static int kfd_parse_subtype_iolink(struct
crat_subtype_iolink *iolink,
props->rec_transfer_size =
iolink->recommended_transfer_size;
- dev->io_link_count++;
dev->node_props.io_links_count++;
list_add_tail(&props->list, &dev->io_link_props);
break;
@@ -1067,7 +1066,6 @@ static int kfd_parse_subtype_iolink(struct
crat_subtype_iolink *iolink,
props2->node_from = id_to;
props2->node_to = id_from;
props2->kobj = NULL;
- to_dev->io_link_count++;
to_dev->node_props.io_links_count++;
list_add_tail(&props2->list, &to_dev->io_link_props);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 3e240b22ec91..304322ac39e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1819,7 +1819,6 @@ static void kfd_topology_update_io_links(int
proximity_domain)
*/
if (iolink->node_to == proximity_domain) {
list_del(&iolink->list);
- dev->io_link_count--;
dev->node_props.io_links_count--;
} else {
if (iolink->node_from > proximity_domain)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index 2fb5664e1041..9f6c949186c1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -130,7 +130,6 @@ struct kfd_topology_device {
struct list_headmem_props;
uint32_tcache_count;
struct list_headcache_props;
- uint32_tio_link_count;
struct list_headio_link_props;
struct list_headp2p_link_props;
struct list_headperf_props;
--
2.35.1
[PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c| 31 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 10 +
5 files changed, 283 insertions(+), 62 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 70a2a0ffc007..d3b2245972b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -223,6 +223,9 @@ static const int __maybe_unused sched_policy =
KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
static const bool __maybe_unused no_system_mem_limit;
#endif
+#ifdef CONFIG_HSA_AMD_P2P
+extern bool pcie_p2p;
+#endif
extern int amdgpu_tmz;
extern int amdgpu_reset_method;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR,/* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..ce012a148ed7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev,
struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment
*attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or
MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO
[PATCH 3/3] drm/amdkfd: Extend KFD device topology to surface peer-to-peer links
Extend KFD device topology to surface peer-to-peer links among
GPU devices connected over PCIe or xGMI. Enabling HSA_AMD_P2P is
REQUIRED to surface peer-to-peer links.
Prior to this KFD did not expose to user mode any P2P links or
indirect links that go over two or more direct hops. Old versions
of the Thunk used to make up their own P2P and indirect links without
the information about peer-accessibility and chipset support available
to the kernel mode driver. In this patch we expose P2P links in a new
sysfs directory to provide more reliable P2P link information to user
mode.
Old versions of the Thunk will continue to work as before and ignore
the new directory. This avoids conflicts between P2P links exposed by
KFD and P2P links created by the Thunk itself. New versions of the Thunk
will use only the P2P links provided in the new p2p_links directory, if
it exists, or fall back to the old code path on older KFDs that don't
expose p2p_links.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 319 +-
drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 3 +
2 files changed, 320 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 8d50d207cf66..3e240b22ec91 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -40,6 +40,7 @@
#include "kfd_svm.h"
#include "amdgpu_amdkfd.h"
#include "amdgpu_ras.h"
+#include "amdgpu.h"
/* topology_device_list - Master list of all topology devices */
static struct list_head topology_device_list;
@@ -148,6 +149,7 @@ static void kfd_release_topology_device(struct
kfd_topology_device *dev)
struct kfd_mem_properties *mem;
struct kfd_cache_properties *cache;
struct kfd_iolink_properties *iolink;
+ struct kfd_iolink_properties *p2plink;
struct kfd_perf_properties *perf;
list_del(&dev->list);
@@ -173,6 +175,13 @@ static void kfd_release_topology_device(struct
kfd_topology_device *dev)
kfree(iolink);
}
+ while (dev->p2p_link_props.next != &dev->p2p_link_props) {
+ p2plink = container_of(dev->p2p_link_props.next,
+ struct kfd_iolink_properties, list);
+ list_del(&p2plink->list);
+ kfree(p2plink);
+ }
+
while (dev->perf_props.next != &dev->perf_props) {
perf = container_of(dev->perf_props.next,
struct kfd_perf_properties, list);
@@ -214,6 +223,7 @@ struct kfd_topology_device *kfd_create_topology_device(
INIT_LIST_HEAD(&dev->mem_props);
INIT_LIST_HEAD(&dev->cache_props);
INIT_LIST_HEAD(&dev->io_link_props);
+ INIT_LIST_HEAD(&dev->p2p_link_props);
INIT_LIST_HEAD(&dev->perf_props);
list_add_tail(&dev->list, device_list);
@@ -465,6 +475,8 @@ static ssize_t node_show(struct kobject *kobj, struct
attribute *attr,
dev->node_props.caches_count);
sysfs_show_32bit_prop(buffer, offs, "io_links_count",
dev->node_props.io_links_count);
+ sysfs_show_32bit_prop(buffer, offs, "p2p_links_count",
+ dev->node_props.p2p_links_count);
sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base",
dev->node_props.cpu_core_id_base);
sysfs_show_32bit_prop(buffer, offs, "simd_id_base",
@@ -568,6 +580,7 @@ static void kfd_remove_sysfs_file(struct kobject *kobj,
struct attribute *attr)
static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
{
+ struct kfd_iolink_properties *p2plink;
struct kfd_iolink_properties *iolink;
struct kfd_cache_properties *cache;
struct kfd_mem_properties *mem;
@@ -585,6 +598,18 @@ static void kfd_remove_sysfs_node_entry(struct
kfd_topology_device *dev)
dev->kobj_iolink = NULL;
}
+ if (dev->kobj_p2plink) {
+ list_for_each_entry(p2plink, &dev->p2p_link_props, list)
+ if (p2plink->kobj) {
+ kfd_remove_sysfs_file(p2plink->kobj,
+ &p2plink->attr);
+ p2plink->kobj = NULL;
+ }
+ kobject_del(dev->kobj_p2plink);
+ kobject_put(dev->kobj_p2plink);
+ dev->kobj_p2plink = NULL;
+ }
+
if (dev->kobj_cache) {
list_for_each_entry(cache, &dev->cache_props, list)
if (cache->kobj) {
@@ -631,6 +656,7 @@ static void kfd_remove_sysfs_node_entry
[PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c| 33 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 280 insertions(+), 62 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR,/* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..ce012a148ed7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev,
struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment
*attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or
MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA
mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it
instead
+ * loops back. The need to DMA map arises only when accessing peer device's
DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without
regard to
+ * device ownership. This is because access requests for MMIO go through PCIe
root
+ * complex.
+ *
+ * This is accomplis
[PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c| 33 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 280 insertions(+), 62 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR,/* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..ce012a148ed7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev,
struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment
*attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or
MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA
mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it
instead
+ * loops back. The need to DMA map arises only when accessing peer device's
DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without
regard to
+ * device ownership. This is because access requests for MMIO go through PCIe
root
+ * complex.
+ *
+ * This is accomplis
[PATCH 3/3] drm/amdkfd: Extend KFD device topology to surface peer-to-peer links
Extend KFD device topology to surface peer-to-peer links among
GPU devices connected over PCIe or xGMI. Enabling HSA_AMD_P2P is
REQUIRED to surface peer-to-peer links.
Prior to this KFD did not expose to user mode any P2P links or
indirect links that go over two or more direct hops. Old versions
of the Thunk used to make up their own P2P and indirect links without
the information about peer-accessibility and chipset support available
to the kernel mode driver. In this patch we expose P2P links in a new
sysfs directory to provide more reliable P2P link information to user
mode.
Old versions of the Thunk will continue to work as before and ignore
the new directory. This avoids conflicts between P2P links exposed by
KFD and P2P links created by the Thunk itself. New versions of the Thunk
will use only the P2P links provided in the new p2p_links directory, if
it exists, or fall back to the old code path on older KFDs that don't
expose p2p_links.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 319 +-
drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 3 +
2 files changed, 320 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 8d50d207cf66..be4dce85e7c1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -40,6 +40,7 @@
#include "kfd_svm.h"
#include "amdgpu_amdkfd.h"
#include "amdgpu_ras.h"
+#include "amdgpu.h"
/* topology_device_list - Master list of all topology devices */
static struct list_head topology_device_list;
@@ -148,6 +149,7 @@ static void kfd_release_topology_device(struct
kfd_topology_device *dev)
struct kfd_mem_properties *mem;
struct kfd_cache_properties *cache;
struct kfd_iolink_properties *iolink;
+ struct kfd_iolink_properties *p2plink;
struct kfd_perf_properties *perf;
list_del(&dev->list);
@@ -173,6 +175,13 @@ static void kfd_release_topology_device(struct
kfd_topology_device *dev)
kfree(iolink);
}
+ while (dev->p2p_link_props.next != &dev->p2p_link_props) {
+ p2plink = container_of(dev->p2p_link_props.next,
+ struct kfd_iolink_properties, list);
+ list_del(&p2plink->list);
+ kfree(p2plink);
+ }
+
while (dev->perf_props.next != &dev->perf_props) {
perf = container_of(dev->perf_props.next,
struct kfd_perf_properties, list);
@@ -214,6 +223,7 @@ struct kfd_topology_device *kfd_create_topology_device(
INIT_LIST_HEAD(&dev->mem_props);
INIT_LIST_HEAD(&dev->cache_props);
INIT_LIST_HEAD(&dev->io_link_props);
+ INIT_LIST_HEAD(&dev->p2p_link_props);
INIT_LIST_HEAD(&dev->perf_props);
list_add_tail(&dev->list, device_list);
@@ -465,6 +475,8 @@ static ssize_t node_show(struct kobject *kobj, struct
attribute *attr,
dev->node_props.caches_count);
sysfs_show_32bit_prop(buffer, offs, "io_links_count",
dev->node_props.io_links_count);
+ sysfs_show_32bit_prop(buffer, offs, "p2p_links_count",
+ dev->node_props.p2p_links_count);
sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base",
dev->node_props.cpu_core_id_base);
sysfs_show_32bit_prop(buffer, offs, "simd_id_base",
@@ -568,6 +580,7 @@ static void kfd_remove_sysfs_file(struct kobject *kobj,
struct attribute *attr)
static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
{
+ struct kfd_iolink_properties *p2plink;
struct kfd_iolink_properties *iolink;
struct kfd_cache_properties *cache;
struct kfd_mem_properties *mem;
@@ -585,6 +598,18 @@ static void kfd_remove_sysfs_node_entry(struct
kfd_topology_device *dev)
dev->kobj_iolink = NULL;
}
+ if (dev->kobj_p2plink) {
+ list_for_each_entry(p2plink, &dev->p2p_link_props, list)
+ if (p2plink->kobj) {
+ kfd_remove_sysfs_file(p2plink->kobj,
+ &p2plink->attr);
+ p2plink->kobj = NULL;
+ }
+ kobject_del(dev->kobj_p2plink);
+ kobject_put(dev->kobj_p2plink);
+ dev->kobj_p2plink = NULL;
+ }
+
if (dev->kobj_cache) {
list_for_each_entry(cache, &dev->cache_props, list)
if (cache->kobj) {
@@ -631,6 +656,7 @@ static void kfd_remove_sysfs_node_entry
[PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 312 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c| 33 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 294 insertions(+), 60 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR,/* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..8e37eae26e49 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev,
struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment
*attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or
MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA
mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it
instead
+ * loops back. The need to DMA map arises only when accessing peer device's
DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without
regard to
+ * device ownership. This is because access requests for MMIO go through PCIe
root
+ * complex.
+ *
+ * This is accomplis
[PATCH 1/3] drm/amdkfd: Define config HSA_AMD_P2P to support peer-to-peer
Extend current kernel config requirements of amdgpu by adding config HSA_AMD_P2P. Enabling HSA_AMD_P2P is REQUIRED to support peer-to-peer communication between AMD GPU devices over PCIe bus Signed-off-by: Ramesh Errabolu --- drivers/gpu/drm/amd/amdkfd/Kconfig | 14 ++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig index 8cc0a76ddf9f..93bd4eda0d94 100644 --- a/drivers/gpu/drm/amd/amdkfd/Kconfig +++ b/drivers/gpu/drm/amd/amdkfd/Kconfig @@ -25,3 +25,17 @@ config HSA_AMD_SVM preemptions and one based on page faults. To enable page fault based memory management on most GFXv9 GPUs, set the module parameter amdgpu.noretry=0. + +config HSA_AMD_P2P + bool "HSA kernel driver support for peer-to-peer for AMD GPU devices" + depends on HSA_AMD && PCI_P2PDMA && DMABUF_MOVE_NOTIFY + help + Enable peer-to-peer (P2P) communication between AMD GPUs over + the PCIe bus. This can improve performance of multi-GPU compute + applications and libraries by enabling GPUs to access data directly + in peer GPUs' memory without intermediate copies in system memory. + + This P2P feature is only enabled on compatible chipsets, and between + GPUs with large memory BARs that expose the entire VRAM in PCIe bus + address space within the physical address limits of the GPUs. + -- 2.35.1
[PATCH 3/3] drm/amdkfd: Extend KFD device topology to surface peer-to-peer links
Extend KFD device topology to surface peer-to-peer links among
GPU devices connected over PCIe or xGMI. Enabling HSA_AMD_P2P is
REQUIRED to surface peer-to-peer links.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 332 +-
drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 3 +
2 files changed, 333 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 8d50d207cf66..6bdc22d6f6ab 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -40,6 +40,7 @@
#include "kfd_svm.h"
#include "amdgpu_amdkfd.h"
#include "amdgpu_ras.h"
+#include "amdgpu.h"
/* topology_device_list - Master list of all topology devices */
static struct list_head topology_device_list;
@@ -148,6 +149,7 @@ static void kfd_release_topology_device(struct
kfd_topology_device *dev)
struct kfd_mem_properties *mem;
struct kfd_cache_properties *cache;
struct kfd_iolink_properties *iolink;
+ struct kfd_iolink_properties *p2plink;
struct kfd_perf_properties *perf;
list_del(&dev->list);
@@ -173,6 +175,13 @@ static void kfd_release_topology_device(struct
kfd_topology_device *dev)
kfree(iolink);
}
+ while (dev->p2p_link_props.next != &dev->p2p_link_props) {
+ p2plink = container_of(dev->p2p_link_props.next,
+ struct kfd_iolink_properties, list);
+ list_del(&p2plink->list);
+ kfree(p2plink);
+ }
+
while (dev->perf_props.next != &dev->perf_props) {
perf = container_of(dev->perf_props.next,
struct kfd_perf_properties, list);
@@ -214,6 +223,7 @@ struct kfd_topology_device *kfd_create_topology_device(
INIT_LIST_HEAD(&dev->mem_props);
INIT_LIST_HEAD(&dev->cache_props);
INIT_LIST_HEAD(&dev->io_link_props);
+ INIT_LIST_HEAD(&dev->p2p_link_props);
INIT_LIST_HEAD(&dev->perf_props);
list_add_tail(&dev->list, device_list);
@@ -465,6 +475,8 @@ static ssize_t node_show(struct kobject *kobj, struct
attribute *attr,
dev->node_props.caches_count);
sysfs_show_32bit_prop(buffer, offs, "io_links_count",
dev->node_props.io_links_count);
+ sysfs_show_32bit_prop(buffer, offs, "p2p_links_count",
+ dev->node_props.p2p_links_count);
sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base",
dev->node_props.cpu_core_id_base);
sysfs_show_32bit_prop(buffer, offs, "simd_id_base",
@@ -568,6 +580,7 @@ static void kfd_remove_sysfs_file(struct kobject *kobj,
struct attribute *attr)
static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
{
+ struct kfd_iolink_properties *p2plink;
struct kfd_iolink_properties *iolink;
struct kfd_cache_properties *cache;
struct kfd_mem_properties *mem;
@@ -585,6 +598,18 @@ static void kfd_remove_sysfs_node_entry(struct
kfd_topology_device *dev)
dev->kobj_iolink = NULL;
}
+ if (dev->kobj_p2plink) {
+ list_for_each_entry(p2plink, &dev->p2p_link_props, list)
+ if (p2plink->kobj) {
+ kfd_remove_sysfs_file(p2plink->kobj,
+ &p2plink->attr);
+ p2plink->kobj = NULL;
+ }
+ kobject_del(dev->kobj_p2plink);
+ kobject_put(dev->kobj_p2plink);
+ dev->kobj_p2plink = NULL;
+ }
+
if (dev->kobj_cache) {
list_for_each_entry(cache, &dev->cache_props, list)
if (cache->kobj) {
@@ -631,6 +656,7 @@ static void kfd_remove_sysfs_node_entry(struct
kfd_topology_device *dev)
static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
uint32_t id)
{
+ struct kfd_iolink_properties *p2plink;
struct kfd_iolink_properties *iolink;
struct kfd_cache_properties *cache;
struct kfd_mem_properties *mem;
@@ -668,6 +694,10 @@ static int kfd_build_sysfs_node_entry(struct
kfd_topology_device *dev,
if (!dev->kobj_iolink)
return -ENOMEM;
+ dev->kobj_p2plink = kobject_create_and_add("p2p_links", dev->kobj_node);
+ if (!dev->kobj_p2plink)
+ return -ENOMEM;
+
dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
if (!dev->kobj_perf)
return -ENOMEM;
@@
[PATCH 1/3] drm/amdkfd: Define config HSA_AMD_P2P to support peer-to-peer
Extend current kernel config requirements of amdgpu by adding config HSA_AMD_P2P. Enabling HSA_AMD_P2P is REQUIRED to support peer-to-peer communication, in both data and control planes, among AMD GPU devices that are connected via PCIe and have large BAR vBIOS Signed-off-by: Ramesh Errabolu --- drivers/gpu/drm/amd/amdkfd/Kconfig | 8 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig index 8cc0a76ddf9f..26614f5f20ea 100644 --- a/drivers/gpu/drm/amd/amdkfd/Kconfig +++ b/drivers/gpu/drm/amd/amdkfd/Kconfig @@ -25,3 +25,11 @@ config HSA_AMD_SVM preemptions and one based on page faults. To enable page fault based memory management on most GFXv9 GPUs, set the module parameter amdgpu.noretry=0. + +config HSA_AMD_P2P + bool "HSA kernel driver support for peer-to-peer for AMD GPU devices" + depends on HSA_AMD && PCI_P2PDMA && DMABUF_MOVE_NOTIFY + help + Enable this if you want to access AMD GPU peer devices, in both data + and control planes, that are connected via PCIe and have large BAR vBIOS + -- 2.35.1
[PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Add support for peer-to-peer communication, in both data and control
planes, among AMD GPUs that are connected PCIe and have large BAR vBIOS.
Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 328 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c| 30 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 307 insertions(+), 60 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR,/* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 34ba9e776521..c2af82317a03 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev,
struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment
*attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or
MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA
mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it
instead
+ * loops back. The need to DMA map arises only when accessing peer device's
DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without
regard to
+ * device ownership. This is because access requests for MMIO
[PATCH] drm/amdgpu: Declare Unpin BO api as static
Fixes warning report from kernel test robot
Reported-by: kernel test robot
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 90b985436878..3463e0d4e5ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1332,7 +1332,7 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo
*bo, u32 domain)
* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
* PIN count decremented. Calls to UNPIN must balance calls to PIN
*/
-void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
+static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
{
int ret = 0;
--
2.31.1
[PATCH 2/2] drm/amdgpu: Pin MMIO/DOORBELL BO's in GTT domain
MMIO/DOORBELL BOs encode control data and should be pinned in GTT
domain before enabling PCIe connected peer devices in accessing it
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 69 +++
1 file changed, 69 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index cfc84af682b1..90b985436878 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1295,6 +1295,55 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void
**process_info,
return ret;
}
+/**
+ * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
+ * @bo: Handle of buffer object being pinned
+ * @domain: Domain into which BO should be pinned
+ *
+ * - USERPTR BOs are UNPINNABLE and will return error
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count incremented. It is valid to PIN a BO multiple times
+ *
+ * Return: ZERO if successful in pinning, Non-Zero in case of error.
+ */
+static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return ret;
+
+ ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
+ if (ret)
+ pr_err("Error in Pinning BO to domain: %d\n", domain);
+
+ amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
+ amdgpu_bo_unreserve(bo);
+
+ return ret;
+}
+
+/**
+ * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
+ * @bo: Handle of buffer object being unpinned
+ *
+ * - Is a illegal request for USERPTR BOs and is ignored
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count decremented. Calls to UNPIN must balance calls to PIN
+ */
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return;
+
+ amdgpu_bo_unpin(bo);
+ amdgpu_bo_unreserve(bo);
+}
+
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
struct file *filp, u32 pasid,
void **process_info,
@@ -1521,10 +1570,22 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
if (offset)
*offset = amdgpu_bo_mmap_offset(bo);
+ if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
+ if (ret) {
+ pr_err("Pinning MMIO/DOORBELL BO during ALLOC
FAILED\n");
+ goto err_pin_bo;
+ }
+ bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
+ bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
+ }
+
return 0;
allocate_init_user_pages_failed:
remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+err_pin_bo:
drm_vma_node_revoke(&gobj->vma_node, drm_priv);
err_node_allow:
drm_gem_object_put(gobj);
@@ -1557,6 +1618,14 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
bool is_imported = false;
mutex_lock(&mem->lock);
+
+ /* Unpin MMIO/DOORBELL BO's that were pinnned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
+ }
+
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
--
2.31.1
[PATCH 1/2] drm/amdgpu: Update BO memory accounting to rely on allocation flag
Accounting system to track amount of available memory (system, TTM
and VRAM of a device) relies on BO's domain. The change is to rely
instead on allocation flag indicating BO type - VRAM, GTT, USERPTR,
MMIO or DOORBELL
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 6 ++
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 95 ---
2 files changed, 65 insertions(+), 36 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index d00de575c541..fcbc8a9c9e06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -301,6 +301,12 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct
amdgpu_device *adev);
void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
struct amdgpu_vm *vm);
+
+/**
+ * @amdgpu_amdkfd_release_notify() - Notify KFD when GEM object is released
+ *
+ * Allows KFD to release its resources associated with the GEM object.
+ */
void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);
void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
#else
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 94fccf0b47ad..cfc84af682b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -120,8 +120,19 @@ static size_t amdgpu_amdkfd_acc_size(uint64_t size)
PAGE_ALIGN(size);
}
+/**
+ * @amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
+ * of buffer including any reserved for control structures
+ *
+ * @adev: Device to which allocated BO belongs to
+ * @size: Size of buffer, in bytes, encapsulated by B0. This should be
+ * equivalent to amdgpu_bo_size(BO)
+ * @alloc_flag: Flag used in allocating a BO as noted above
+ *
+ * Return: returns -ENOMEM in case of error, ZERO otherwise
+ */
static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
- uint64_t size, u32 domain, bool sg)
+ uint64_t size, u32 alloc_flag)
{
uint64_t reserved_for_pt =
ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
@@ -131,20 +142,24 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct
amdgpu_device *adev,
acc_size = amdgpu_amdkfd_acc_size(size);
vram_needed = 0;
- if (domain == AMDGPU_GEM_DOMAIN_GTT) {
- /* TTM GTT memory */
+ if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
system_mem_needed = acc_size + size;
ttm_mem_needed = acc_size + size;
- } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
- /* Userptr */
+ } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
+ system_mem_needed = acc_size;
+ ttm_mem_needed = acc_size;
+ vram_needed = size;
+ } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
system_mem_needed = acc_size + size;
ttm_mem_needed = acc_size;
- } else {
- /* VRAM and SG */
+ } else if (alloc_flag &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
system_mem_needed = acc_size;
ttm_mem_needed = acc_size;
- if (domain == AMDGPU_GEM_DOMAIN_VRAM)
- vram_needed = size;
+ } else {
+ pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
+ return -ENOMEM;
}
spin_lock(&kfd_mem_limit.mem_limit_lock);
@@ -160,64 +175,72 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct
amdgpu_device *adev,
(adev->kfd.vram_used + vram_needed >
adev->gmc.real_vram_size - reserved_for_pt)) {
ret = -ENOMEM;
- } else {
- kfd_mem_limit.system_mem_used += system_mem_needed;
- kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
- adev->kfd.vram_used += vram_needed;
+ goto release;
}
+ /* Update memory accounting by decreasing available system
+* memory, TTM memory and GPU memory as computed above
+*/
+ adev->kfd.vram_used += vram_needed;
+ kfd_mem_limit.system_mem_used += system_mem_needed;
+ kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
+
+release:
spin_unlock(&kfd_mem_limit.mem_limit_lock);
return ret;
}
static void unreserve_mem_limit(struct amdgpu_device *adev,
- uint64_t size, u32 domain, bool sg)
+ uint64_t size, u32 alloc_flag)
{
size_t acc_size;
acc_size = amdgpu_amdkfd_acc_size(size);
spin_lock(&kfd_mem_limit.mem_limit_lock);
- if (domain == AMDGPU_GEM_DO
[PATCH] drm/amdgpu: Pin MMIO/DOORBELL BO's in GTT domain
MMIO/DOORBELL BOs encode control data and should be pinned in GTT
domain before enabling PCIe connected peer devices in accessing it
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 70 +++
1 file changed, 70 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 08675f89bfb2..5e063fac0250 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1297,6 +1297,56 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void
**process_info,
return ret;
}
+/**
+ * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
+ * @bo: Handle of buffer object being pinned
+ * @domain: Domain into which BO should be pinned
+ *
+ * - USERPTR BOs are UNPINNABLE and will return error
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count incremented. It is valid to PIN a BO multiple times
+ *
+ * Return: ZERO if successful in pinning, Non-Zero in case of error.
+ * Will return -EINVAL if input BO parameter is a USERPTR type.
+ */
+static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return ret;
+
+ ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
+ if (ret)
+ pr_err("Error in Pinning BO to domain: %d\n", domain);
+
+ amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
+ amdgpu_bo_unreserve(bo);
+
+ return ret;
+}
+
+/**
+ * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
+ * @bo: Handle of buffer object being unpinned
+ *
+ * - Is a illegal request for USERPTR BOs and is ignored
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count decremented. Calls to UNPIN must balance calls to PIN
+ */
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return;
+
+ amdgpu_bo_unpin(bo);
+ amdgpu_bo_unreserve(bo);
+}
+
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
struct file *filp, u32 pasid,
void **process_info,
@@ -1523,10 +1573,22 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
if (offset)
*offset = amdgpu_bo_mmap_offset(bo);
+ if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
+ if (ret) {
+ pr_err("Pinning MMIO/DOORBELL BO during ALLOC
FAILED\n");
+ goto err_pin_bo;
+ }
+ bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
+ bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
+ }
+
return 0;
allocate_init_user_pages_failed:
remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+err_pin_bo:
drm_vma_node_revoke(&gobj->vma_node, drm_priv);
err_node_allow:
drm_gem_object_put(gobj);
@@ -1559,6 +1621,14 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
bool is_imported = false;
mutex_lock(&mem->lock);
+
+ /* Unpin MMIO/DOORBELL BO's that were pinnned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
+ }
+
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
--
2.31.1
[PATCH 1/2] drm/amdgpu: Update BO memory accounting to rely on allocation flag
Accounting system to track amount of available memory (system, TTM
and VRAM of a device) relies on BO's domain. The change is to rely
instead on allocation flag indicating BO type - VRAM, GTT, USERPTR,
MMIO or DOORBELL
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 16 +++
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 101 +++---
2 files changed, 79 insertions(+), 38 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 5f658823a637..8d31a742cd80 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -307,7 +307,23 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct
amdgpu_device *adev);
void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
struct amdgpu_vm *vm);
+
+/**
+ * @amdgpu_amdkfd_release_notify() - Invoked when GEM object reference count
+ * reaches ZERO. Increases available memory by size of buffer including any
+ * reserved for control structures
+ *
+ * @note: This api must be invoked on BOs that have been allocated via
+ * KFD interface amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
+ */
void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);
+
+/**
+ * @amdgpu_amdkfd_reserve_system_mem - Decrease system memory that is
+ * available by an amount specified by input parameter
+ *
+ * @size: Size of buffer in bytes
+ */
void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
#else
static inline
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 94fccf0b47ad..08675f89bfb2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -120,8 +120,22 @@ static size_t amdgpu_amdkfd_acc_size(uint64_t size)
PAGE_ALIGN(size);
}
+/**
+ * @amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
+ * of buffer including any reserved for control structures
+ *
+ * @adev: Device to which allocated BO belongs to
+ * @size: Size of buffer, in bytes, encapsulated by B0. This should be
+ * equivalent to amdgpu_bo_size(BO)
+ * @alloc_flag: Flag used in allocating a BO as noted above
+ *
+ * @note: This api must be invoked on BOs that have been allocated via
+ * KFD interface amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
+ *
+ * Return: returns -ENOMEM in case of error, ZERO otherwise
+ */
static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
- uint64_t size, u32 domain, bool sg)
+ uint64_t size, u32 alloc_flag)
{
uint64_t reserved_for_pt =
ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
@@ -131,20 +145,24 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct
amdgpu_device *adev,
acc_size = amdgpu_amdkfd_acc_size(size);
vram_needed = 0;
- if (domain == AMDGPU_GEM_DOMAIN_GTT) {
- /* TTM GTT memory */
+ if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
system_mem_needed = acc_size + size;
ttm_mem_needed = acc_size + size;
- } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
- /* Userptr */
+ } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
+ system_mem_needed = acc_size;
+ ttm_mem_needed = acc_size;
+ vram_needed = size;
+ } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
system_mem_needed = acc_size + size;
ttm_mem_needed = acc_size;
- } else {
- /* VRAM and SG */
+ } else if (alloc_flag &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
system_mem_needed = acc_size;
ttm_mem_needed = acc_size;
- if (domain == AMDGPU_GEM_DOMAIN_VRAM)
- vram_needed = size;
+ } else {
+ pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
+ return -ENOMEM;
}
spin_lock(&kfd_mem_limit.mem_limit_lock);
@@ -160,64 +178,71 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct
amdgpu_device *adev,
(adev->kfd.vram_used + vram_needed >
adev->gmc.real_vram_size - reserved_for_pt)) {
ret = -ENOMEM;
- } else {
- kfd_mem_limit.system_mem_used += system_mem_needed;
- kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
- adev->kfd.vram_used += vram_needed;
+ goto release;
}
+ /* Update memory accounting by decreasing available system
+* memory, TTM memory and GPU memory as computed above
+*/
+ adev->kfd.vram_used += vram_needed;
[PATCH 2/2] drm/amdgpu: Pin MMIO/DOORBELL BO's in GTT domain
MMIO/DOORBELL BOs encode control data and should be pinned in GTT
domain before enabling PCIe connected peer devices in accessing it
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 76 +++
1 file changed, 76 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 08675f89bfb2..adc27c0031b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1297,6 +1297,56 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void
**process_info,
return ret;
}
+/**
+ * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
+ * @bo: Handle of buffer object being pinned
+ * @domain: Domain into which BO should be pinned
+ *
+ * - USERPTR BOs are UNPINNABLE and will return error
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count incremented. It is valid to PIN a BO multiple times
+ *
+ * Return: ZERO if successful in pinning, Non-Zero in case of error.
+ * Will return -EINVAL if input BO parameter is a USERPTR type.
+ */
+static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return ret;
+
+ ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
+ if (ret)
+ pr_err("Error in Pinning BO to domain: %d\n", domain);
+
+ amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
+ amdgpu_bo_unreserve(bo);
+
+ return ret;
+}
+
+/**
+ * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
+ * @bo: Handle of buffer object being unpinned
+ *
+ * - Is a illegal request for USERPTR BOs and is ignored
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count decremented. Calls to UNPIN must balance calls to PIN
+ */
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return;
+
+ amdgpu_bo_unpin(bo);
+ amdgpu_bo_unreserve(bo);
+}
+
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
struct file *filp, u32 pasid,
void **process_info,
@@ -1523,10 +1573,28 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
if (offset)
*offset = amdgpu_bo_mmap_offset(bo);
+ if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ ret = amdgpu_amdkfd_bo_validate(bo, AMDGPU_GEM_DOMAIN_GTT,
false);
+ if (ret) {
+ pr_err("Validating MMIO/DOORBELL BO during ALLOC
FAILED\n");
+ goto err_pin_bo;
+ }
+
+ ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
+ if (ret) {
+ pr_err("Pinning MMIO/DOORBELL BO during ALLOC
FAILED\n");
+ goto err_pin_bo;
+ }
+ bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
+ bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
+ }
+
return 0;
allocate_init_user_pages_failed:
remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+err_pin_bo:
drm_vma_node_revoke(&gobj->vma_node, drm_priv);
err_node_allow:
drm_gem_object_put(gobj);
@@ -1559,6 +1627,14 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
bool is_imported = false;
mutex_lock(&mem->lock);
+
+ /* Unpin MMIO/DOORBELL BO's that were pinnned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
+ }
+
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
--
2.31.1
[PATCH] drm/amdgpu: Pin MMIO/DOORBELL BO's in GTT domain
MMIO/DOORBELL BOs encode control data and should be pinned in GTT
domain before enabling PCIe connected peer devices in accessing it
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 25 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 55 +++
2 files changed, 80 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4c1d6536a7a5..d9a1cfd7876f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -300,6 +300,31 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device
*adev,
uint64_t va, void *drm_priv,
struct kgd_mem **mem, uint64_t *size,
uint64_t *mmap_offset);
+
+/**
+ * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
+ * @bo: Handle of buffer object being pinned
+ * @domain: Domain into which BO should be pinned
+ *
+ * - USERPTR BOs are UNPINNABLE and will return error
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count incremented. It is valid to PIN a BO multiple times
+ *
+ * Return: ZERO if successful in pinning, Non-Zero in case of error.
+ * Will return -EINVAL if input BO parameter is a USERPTR type.
+ */
+int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain);
+
+/**
+ * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
+ * @bo: Handle of buffer object being unpinned
+ *
+ * - Is a illegal request for USERPTR BOs and is ignored
+ * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
+ * PIN count decremented. Calls to UNPIN must balance calls to PIN
+ */
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo);
+
int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
struct tile_config *config);
void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 4fa814358552..f4ffc41873dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1299,6 +1299,36 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void
**process_info,
return ret;
}
+int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return ret;
+
+ ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
+ if (ret)
+ pr_err("Error in Pinning BO to domain: %d\n", domain);
+
+ amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
+ amdgpu_bo_unreserve(bo);
+
+ return ret;
+}
+
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
+{
+ int ret = 0;
+
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret))
+ return;
+
+ amdgpu_bo_unpin(bo);
+ amdgpu_bo_unreserve(bo);
+}
+
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
struct file *filp, u32 pasid,
void **process_info,
@@ -1525,6 +1555,23 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
if (offset)
*offset = amdgpu_bo_mmap_offset(bo);
+ if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ ret = amdgpu_amdkfd_bo_validate(bo, AMDGPU_GEM_DOMAIN_GTT,
false);
+ if (ret) {
+ pr_err("Validating MMIO/DOORBELL BO during ALLOC
FAILED\n");
+ goto err_node_allow;
+ }
+
+ ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
+ if (ret) {
+ pr_err("Pinning MMIO/DOORBELL BO during ALLOC
FAILED\n");
+ goto err_node_allow;
+ }
+ bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
+ bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
+ }
+
return 0;
allocate_init_user_pages_failed:
@@ -1561,6 +1608,14 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
bool is_imported = false;
mutex_lock(&mem->lock);
+
+ /* Unpin MMIO/DOORBELL BO's that were pinnned during allocation */
+ if (mem->alloc_flags &
+ (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+ amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
+ }
+
mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
is_imported = mem->is_imported;
mutex_unlock(&mem->lock);
--
2.31.1
[PATCH] Whitelist AMD host bridge device(s) to enable P2P DMA
Current implementation will disallow P2P DMA if the participating
devices belong to different root complexes. Implementation allows
this default behavior to be overridden for whitelisted devices. The
patch adds an AMD host bridge to be whitelisted
Signed-off-by: Ramesh Errabolu
---
drivers/pci/p2pdma.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 196382630363..7003bb9faf23 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -305,6 +305,8 @@ static const struct pci_p2pdma_whitelist_entry {
{PCI_VENDOR_ID_INTEL, 0x2032, 0},
{PCI_VENDOR_ID_INTEL, 0x2033, 0},
{PCI_VENDOR_ID_INTEL, 0x2020, 0},
+ /* AMD Host Bridge Devices */
+ {PCI_VENDOR_ID_AMD, 0x1480, 0},
{}
};
--
2.31.1
[PATCH] drm/amdgpu: Use iterator methods exposed by amdgpu_res_cursor.h in building SG_TABLE's for a VRAM BO
Extend current implementation of SG_TABLE construction method to
allow exportation of sub-buffers of a VRAM BO. This capability will
enable logical partitioning of a VRAM BO into multiple non-overlapping
sub-buffers. One example of this use case is to partition a VRAM BO
into two sub-buffers, one for SRC and another for DST.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 +--
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 34 ++--
3 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index e0c4f7c7f1b9..baa980a477d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -291,8 +291,8 @@ static struct sg_table *amdgpu_dma_buf_map(struct
dma_buf_attachment *attach,
break;
case TTM_PL_VRAM:
- r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, attach->dev,
- dir, &sgt);
+ r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, 0,
+ bo->tbo.base.size, attach->dev, dir, &sgt);
if (r)
return ERR_PTR(r);
break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index dec0db8b0b13..9e38475e0f8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -112,6 +112,7 @@ int amdgpu_gtt_mgr_recover(struct ttm_resource_manager
*man);
u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ u64 offset, u64 size,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 592a2dd16493..bce105e2973e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -25,6 +25,7 @@
#include
#include "amdgpu.h"
#include "amdgpu_vm.h"
+#include "amdgpu_res_cursor.h"
#include "amdgpu_atomfirmware.h"
#include "atom.h"
@@ -565,6 +566,8 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager
*man,
*
* @adev: amdgpu device pointer
* @mem: TTM memory object
+ * @offset: byte offset from the base of VRAM BO
+ * @length: number of bytes to export in sg_table
* @dev: the other device
* @dir: dma direction
* @sgt: resulting sg table
@@ -573,39 +576,47 @@ static void amdgpu_vram_mgr_del(struct
ttm_resource_manager *man,
*/
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ u64 offset, u64 length,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt)
{
- struct drm_mm_node *node;
+ struct amdgpu_res_cursor cursor;
struct scatterlist *sg;
int num_entries = 0;
- unsigned int pages;
int i, r;
*sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
if (!*sgt)
return -ENOMEM;
- for (pages = mem->num_pages, node = mem->mm_node;
-pages; pages -= node->size, ++node)
- ++num_entries;
+ /* Determine the number of DRM_MM nodes to export */
+ amdgpu_res_first(mem, offset, length, &cursor);
+ while (cursor.remaining) {
+ num_entries++;
+ amdgpu_res_next(&cursor, cursor.size);
+ }
r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
if (r)
goto error_free;
+ /* Initialize scatterlist nodes of sg_table */
for_each_sgtable_sg((*sgt), sg, i)
sg->length = 0;
- node = mem->mm_node;
+ /*
+* Walk down DRM_MM nodes to populate scatterlist nodes
+* @note: Use iterator api to get first the DRM_MM node
+* and the number of bytes from it. Access the following
+* DRM_MM node(s) if more buffer needs to exported
+*/
+ amdgpu_res_first(mem, offset, length, &cursor);
for_each_sgtable_sg((*sgt), sg, i) {
- phys_addr_t phys = (node->start << PAGE_SHIFT) +
- adev->gmc.aper_base;
- size_t size = node->size << PAGE_SHIFT;
+ phys_addr_t phys = cursor.start + adev->gmc.aper_base;
+ size_t size =
[PATCH] drm/amdgpu: Use iterator methods exposed by amdgpu_res_cursor.h in building SG_TABLE's for a VRAM BO
Extend current implementation of SG_TABLE construction method to
allow exportation of sub-buffers of a VRAM BO. This capability will
enable logical partitioning of a VRAM BO into multiple non-overlapping
sub-buffers. One example of this use case is to partition a VRAM BO
into two sub-buffers, one for SRC and another for DST.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 6 ++--
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 34 ++--
3 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index e0c4f7c7f1b9..57534b93bd91 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -255,6 +255,7 @@ static struct sg_table *amdgpu_dma_buf_map(struct
dma_buf_attachment *attach,
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
struct sg_table *sgt;
+ u64 num_bytes;
long r;
if (!bo->tbo.pin_count) {
@@ -291,8 +292,9 @@ static struct sg_table *amdgpu_dma_buf_map(struct
dma_buf_attachment *attach,
break;
case TTM_PL_VRAM:
- r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, attach->dev,
- dir, &sgt);
+ num_bytes = bo->tbo.mem.num_pages << PAGE_SHIFT;
+ r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, 0, num_bytes,
+ attach->dev, dir, &sgt);
if (r)
return ERR_PTR(r);
break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index dec0db8b0b13..9e38475e0f8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -112,6 +112,7 @@ int amdgpu_gtt_mgr_recover(struct ttm_resource_manager
*man);
u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ u64 offset, u64 size,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 592a2dd16493..c1a7772fa8e8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -25,6 +25,7 @@
#include
#include "amdgpu.h"
#include "amdgpu_vm.h"
+#include "amdgpu_res_cursor.h"
#include "amdgpu_atomfirmware.h"
#include "atom.h"
@@ -565,6 +566,8 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager
*man,
*
* @adev: amdgpu device pointer
* @mem: TTM memory object
+ * @offset: byte offset from the base of VRAM BO
+ * @length: number of bytes to export in sg_table
* @dev: the other device
* @dir: dma direction
* @sgt: resulting sg table
@@ -573,39 +576,47 @@ static void amdgpu_vram_mgr_del(struct
ttm_resource_manager *man,
*/
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ u64 offset, u64 length,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt)
{
- struct drm_mm_node *node;
+ struct amdgpu_res_cursor cursor;
struct scatterlist *sg;
int num_entries = 0;
- unsigned int pages;
int i, r;
*sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
if (!*sgt)
return -ENOMEM;
- for (pages = mem->num_pages, node = mem->mm_node;
-pages; pages -= node->size, ++node)
- ++num_entries;
+ /* Determine the number of DRM_MM nodes to export */
+ amdgpu_res_first(mem, offset, length, &cursor);
+ while (cursor.remaining) {
+ num_entries++;
+ amdgpu_res_next(&cursor, cursor.size);
+ }
r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
if (r)
goto error_free;
+ /* Initialize scatterlist nodes of sg_table */
for_each_sgtable_sg((*sgt), sg, i)
sg->length = 0;
- node = mem->mm_node;
+ /*
+* Walk down DRM_MM nodes to populate scatterlist nodes
+* @note: Use iterator api to get first the DRM_MM node
+* and the number of bytes from it. Access the following
+* DRM_MM node(s) if more buffer needs to exported
+*/
+ amdgpu_res_first(
[PATCH] drm/amdgpu: Use iterator methods exposed by amdgpu_res_cursor.h in building SG_TABLE's for a VRAM BO
Extend current implementation of SG_TABLE construction method to
allow exportation of sub-buffers of a VRAM BO. This capability will
enable logical partitioning of a VRAM BO into multiple non-overlapping
sub-buffers. One example of this use case is to partition a VRAM BO
into two sub-buffers, one for SRC and another for DST.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 6 ++--
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 32 ++--
3 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index e0c4f7c7f1b9..57534b93bd91 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -255,6 +255,7 @@ static struct sg_table *amdgpu_dma_buf_map(struct
dma_buf_attachment *attach,
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
struct sg_table *sgt;
+ u64 num_bytes;
long r;
if (!bo->tbo.pin_count) {
@@ -291,8 +292,9 @@ static struct sg_table *amdgpu_dma_buf_map(struct
dma_buf_attachment *attach,
break;
case TTM_PL_VRAM:
- r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, attach->dev,
- dir, &sgt);
+ num_bytes = bo->tbo.mem.num_pages << PAGE_SHIFT;
+ r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, 0, num_bytes,
+ attach->dev, dir, &sgt);
if (r)
return ERR_PTR(r);
break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index dec0db8b0b13..9e38475e0f8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -112,6 +112,7 @@ int amdgpu_gtt_mgr_recover(struct ttm_resource_manager
*man);
u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ u64 offset, u64 size,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 592a2dd16493..fcdee0deba16 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -25,6 +25,7 @@
#include
#include "amdgpu.h"
#include "amdgpu_vm.h"
+#include "amdgpu_res_cursor.h"
#include "amdgpu_atomfirmware.h"
#include "atom.h"
@@ -565,6 +566,8 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager
*man,
*
* @adev: amdgpu device pointer
* @mem: TTM memory object
+ * @offset: byte offset from the base of VRAM BO
+ * @length: number of bytes to export in sg_table
* @dev: the other device
* @dir: dma direction
* @sgt: resulting sg table
@@ -573,39 +576,45 @@ static void amdgpu_vram_mgr_del(struct
ttm_resource_manager *man,
*/
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ u64 offset, u64 length,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt)
{
- struct drm_mm_node *node;
+ struct amdgpu_res_cursor cursor;
struct scatterlist *sg;
int num_entries = 0;
- unsigned int pages;
int i, r;
*sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
if (!*sgt)
return -ENOMEM;
- for (pages = mem->num_pages, node = mem->mm_node;
-pages; pages -= node->size, ++node)
- ++num_entries;
+ // Determine the number of DRM_MM nodes to export
+ amdgpu_res_first(mem, offset, length, &cursor);
+ while (cursor.remaining) {
+ num_entries++;
+ amdgpu_res_next(&cursor, cursor.size);
+ }
r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
if (r)
goto error_free;
+ // Initialize scatterlist nodes of sg_table
for_each_sgtable_sg((*sgt), sg, i)
sg->length = 0;
- node = mem->mm_node;
+ // Walk down DRM_MM nodes to populate scatterlist nodes
+ // @note: Use iterator api to get first the DRM_MM node
+ // and the number of bytes from it. Access the following
+ // DRM_MM node(s) if more buffer needs to exported
+ amdgpu_res_first(mem, offset, length,
[PATCH] drm/amdgpu: Remove amdgpu_device arg from free_sgt api
Currently callers have to provide handle of amdgpu_device,
which is not used by the implementation. It is unlikely this
parameter will become useful in future, thus removing it
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 3 +--
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 +--
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 4 +---
3 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index 2808d5752de1..e83d73ec0e9d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -359,14 +359,13 @@ static void amdgpu_dma_buf_unmap(struct
dma_buf_attachment *attach,
struct dma_buf *dma_buf = attach->dmabuf;
struct drm_gem_object *obj = dma_buf->priv;
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
- struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
if (sgt->sgl->page_link) {
dma_unmap_sgtable(attach->dev, sgt, dir, 0);
sg_free_table(sgt);
kfree(sgt);
} else {
- amdgpu_vram_mgr_free_sgt(adev, attach->dev, dir, sgt);
+ amdgpu_vram_mgr_free_sgt(attach->dev, dir, sgt);
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 927d33d75c22..028f200a3509 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -121,8 +121,7 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt);
-void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
- struct device *dev,
+void amdgpu_vram_mgr_free_sgt(struct device *dev,
enum dma_data_direction dir,
struct sg_table *sgt);
uint64_t amdgpu_vram_mgr_usage(struct ttm_resource_manager *man);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index b325b067926b..14936bc713b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -734,15 +734,13 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
/**
* amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
*
- * @adev: amdgpu device pointer
* @dev: device pointer
* @dir: data direction of resource to unmap
* @sgt: sg table to free
*
* Free a previously allocate sg table.
*/
-void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
- struct device *dev,
+void amdgpu_vram_mgr_free_sgt(struct device *dev,
enum dma_data_direction dir,
struct sg_table *sgt)
{
--
2.30.0
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH] drm/amdgpu: Modify alloc_sgt apis to consider size of request
The current method that builds SG table does not allow its users
to request a sub-block of the buffer object. The change modifies
api signature to allow users to specify both the offset and size
of the request.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 11 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 8 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 9 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 245 +++
4 files changed, 210 insertions(+), 63 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index 2808d5752de1..b23f44999814 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -326,8 +326,8 @@ static struct sg_table *amdgpu_dma_buf_map(struct
dma_buf_attachment *attach,
break;
case TTM_PL_VRAM:
- r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, attach->dev,
- dir, &sgt);
+ r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, 0, 0,
+ attach->dev, dir, &sgt);
if (r)
return ERR_PTR(r);
break;
@@ -356,17 +356,12 @@ static void amdgpu_dma_buf_unmap(struct
dma_buf_attachment *attach,
struct sg_table *sgt,
enum dma_data_direction dir)
{
- struct dma_buf *dma_buf = attach->dmabuf;
- struct drm_gem_object *obj = dma_buf->priv;
- struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
- struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-
if (sgt->sgl->page_link) {
dma_unmap_sgtable(attach->dev, sgt, dir, 0);
sg_free_table(sgt);
kfree(sgt);
} else {
- amdgpu_vram_mgr_free_sgt(adev, attach->dev, dir, sgt);
+ amdgpu_vram_mgr_free_sgt(attach->dev, dir, sgt);
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index ce92768cd146..ad504d0e5b26 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -222,8 +222,8 @@ static uint64_t amdgpu_mm_node_addr(struct
ttm_buffer_object *bo,
* @offset: The offset that drm_mm_node is used for finding.
*
*/
-static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_resource *mem,
- uint64_t *offset)
+struct drm_mm_node *amdgpu_find_mm_node(struct ttm_resource *mem,
+ uint64_t *offset)
{
struct drm_mm_node *mm_node = mem->mm_node;
@@ -782,8 +782,8 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device
*bdev, struct ttm_reso
return 0;
}
-static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
- unsigned long page_offset)
+unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
+ unsigned long page_offset)
{
uint64_t offset = (page_offset << PAGE_SHIFT);
struct drm_mm_node *mm;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 4df4cf2fd4dd..e3e413dbfd72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -116,11 +116,12 @@ int amdgpu_gtt_mgr_recover(struct ttm_resource_manager
*man);
u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
struct ttm_resource *mem,
+ uint64_t req_offset,
+ uint64_t req_size,
struct device *dev,
enum dma_data_direction dir,
struct sg_table **sgt);
-void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
- struct device *dev,
+void amdgpu_vram_mgr_free_sgt(struct device *dev,
enum dma_data_direction dir,
struct sg_table *sgt);
uint64_t amdgpu_vram_mgr_usage(struct ttm_resource_manager *man);
@@ -155,6 +156,10 @@ int amdgpu_mmap(struct file *filp, struct vm_area_struct
*vma);
int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type);
+struct drm_mm_node *amdgpu_find_mm_node(struct ttm_resource *mem,
+ uint64_t *offset);
+unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
+ unsigned long page_offset);
#if IS_ENABLED(CONFIG_DRM_AMDGPU_US
[PATCH 2/2] drm/amdkfd: Update RDMA implementation such that get_pages() only pins memory of a BO while dma_map() builds an sg_table from the pinned memory
[Why]
Enable RDMA service when platform has IOMMU ON. The handle of device
to DMA map is available only in dma_map() call context.
[How]
Update implementation get_pages() such that it only Pins memory of a
buffer object. Update implementation dma_map() to use the handle of
DMA device in building an sg_table off of the buffer object
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c | 47 +++--
drivers/gpu/drm/amd/amdkfd/kfd_rdma.c | 26 +++-
2 files changed, 59 insertions(+), 14 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
b/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
index 6d7340b101ba..86b4ee710629 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
@@ -54,7 +54,6 @@
#include "kfd_priv.h"
-
/* --- PeerDirect interface
--*/
/*
@@ -281,6 +280,14 @@ static int amd_get_pages(unsigned long addr, size_t size,
int write, int force,
static int amd_dma_map(struct sg_table *sg_head, void *client_context,
struct device *dma_device, int dmasync, int *nmap)
{
+ struct sg_table *sg_table_tmp;
+ struct kfd_bo *buf_obj;
+ struct kfd_dev *dev;
+ struct kgd_mem *mem;
+ uint64_t offset;
+ uint64_t length;
+ int ret;
+
/*
* NOTE/TODO:
* We could have potentially three cases for real memory
@@ -312,15 +319,33 @@ static int amd_dma_map(struct sg_table *sg_head, void
*client_context,
mem_context->size);
if (!mem_context->p2p_info) {
- pr_err("No sg table were allocated\n");
+ pr_err("Buffer was not pinned\n");
return -EINVAL;
}
+ /* Retrieve the handle of buffer object embedded in amd_p2p_info */
+ buf_obj = mem_context->p2p_info->priv;
+ mem = buf_obj->mem;
+ dev = buf_obj->dev;
+ offset = mem_context->va - buf_obj->it.start;
+ length = mem_context->p2p_info->size;
+
+ /* Build sg_table for buffer being exported, including DMA mapping */
+ ret = amdgpu_amdkfd_gpuvm_get_sg_table(dev->kgd, mem,
+ offset, length, &sg_table_tmp);
+ if (ret) {
+ pr_err("Building of sg_table failed\n");
+ return -EFAULT;
+ }
+
+ /* Maintain a copy of the handle to sg_table */
+ mem_context->p2p_info->pages = sg_table_tmp;
+
/* Copy information about previosly allocated sg_table */
- *sg_head = *mem_context->p2p_info->pages;
+ *sg_head = *sg_table_tmp;
/* Return number of pages */
- *nmap = mem_context->p2p_info->pages->nents;
+ *nmap = sg_table_tmp->nents;
return 0;
}
@@ -328,6 +353,7 @@ static int amd_dma_map(struct sg_table *sg_head, void
*client_context,
static int amd_dma_unmap(struct sg_table *sg_head, void *client_context,
struct device *dma_device)
{
+ struct sg_table *sg_table_tmp;
struct amd_mem_context *mem_context =
(struct amd_mem_context *)client_context;
@@ -339,6 +365,19 @@ static int amd_dma_unmap(struct sg_table *sg_head, void
*client_context,
mem_context->va,
mem_context->size);
+ /* Determine if sg_table construction was successful */
+ if (!mem_context->p2p_info) {
+ pr_err("Buffer was not pinned\n");
+ return -EINVAL;
+ }
+
+ /* Free sg_table and its elements including DMA unmapping */
+ sg_table_tmp = mem_context->p2p_info->pages;
+ if (sg_table_tmp != NULL) {
+ mem_context->p2p_info->pages = NULL;
+ amdgpu_amdkfd_gpuvm_put_sg_table(sg_table_tmp);
+ }
+
/* Assume success */
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
b/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
index ec6435aa33bd..67a9213a40b3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
@@ -66,7 +66,6 @@ static int get_pages(uint64_t address, uint64_t length,
struct pid *pid,
{
struct kfd_bo *buf_obj;
struct kgd_mem *mem;
- struct sg_table *sg_table_tmp;
struct kfd_dev *dev;
uint64_t last = address + length - 1;
uint64_t offset;
@@ -99,10 +98,9 @@ static int get_pages(uint64_t address, uint64_t length,
struct pid *pid,
dev = buf_obj->dev;
offset = address - buf_obj->it.start;
- ret = amdgpu_amdkfd_gpuvm_pin_get_sg_table(dev->kgd, mem,
- offset, length, &sg_table_tmp);
+ ret = amdgpu_amdkfd_gpuvm_pin_bo(mem);
if (ret) {
- pr_err("amdgpu_amdkfd_gpuvm_pi
[PATCH 1/2] drm/amdgpu: Enable pinning of a BO and building an sg_table off of it as separate peer services
[Why]
Enable users to determine the use and sequencing of these services.
Not all uses of pinning need to be followed by building of sg_table
[How]
Rename and refactor the implementaions of gpuvm_pin_get_sgtable() and
gpuvm_unpin_put_sgtable() apis in gpuvm.c
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h| 12 ++
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 23 ---
2 files changed, 12 insertions(+), 23 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 8013470d7c8d..92860b16c14b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -249,11 +249,13 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void
*process_info,
int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
struct kfd_vm_fault_info *info);
-int amdgpu_amdkfd_gpuvm_pin_get_sg_table(struct kgd_dev *kgd,
- struct kgd_mem *mem, uint64_t offset,
- uint64_t size, struct sg_table **ret_sg);
-void amdgpu_amdkfd_gpuvm_unpin_put_sg_table(
- struct kgd_mem *mem, struct sg_table *sg);
+int amdgpu_amdkfd_gpuvm_pin_bo(struct kgd_mem *mem);
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct kgd_mem *mem);
+
+int amdgpu_amdkfd_gpuvm_get_sg_table(struct kgd_dev *kgd, struct kgd_mem *mem,
+ uint64_t offset, uint64_t size, struct sg_table **ret_sg);
+void amdgpu_amdkfd_gpuvm_put_sg_table(struct sg_table *sg);
+
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
struct dma_buf *dmabuf,
struct kfd_ipc_obj *ipc_obj,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 702cad576728..0d1984357451 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1726,7 +1726,7 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev
*kgd,
return 0;
}
-static int pin_bo_wo_map(struct kgd_mem *mem)
+int amdgpu_amdkfd_gpuvm_pin_bo(struct kgd_mem *mem)
{
struct amdgpu_bo *bo = mem->bo;
int ret = 0;
@@ -1742,7 +1742,7 @@ static int pin_bo_wo_map(struct kgd_mem *mem)
return ret;
}
-static void unpin_bo_wo_map(struct kgd_mem *mem)
+void amdgpu_amdkfd_gpuvm_unpin_bo(struct kgd_mem *mem)
{
struct amdgpu_bo *bo = mem->bo;
int ret = 0;
@@ -1852,33 +1852,20 @@ static int get_sg_table(struct amdgpu_device *adev,
return ret;
}
-int amdgpu_amdkfd_gpuvm_pin_get_sg_table(struct kgd_dev *kgd,
+int amdgpu_amdkfd_gpuvm_get_sg_table(struct kgd_dev *kgd,
struct kgd_mem *mem, uint64_t offset,
uint64_t size, struct sg_table **ret_sg)
{
- int ret;
struct amdgpu_device *adev;
- ret = pin_bo_wo_map(mem);
- if (unlikely(ret))
- return ret;
-
adev = get_amdgpu_device(kgd);
-
- ret = get_sg_table(adev, mem, offset, size, ret_sg);
- if (ret)
- unpin_bo_wo_map(mem);
-
- return ret;
+ return get_sg_table(adev, mem, offset, size, ret_sg);
}
-void amdgpu_amdkfd_gpuvm_unpin_put_sg_table(
- struct kgd_mem *mem, struct sg_table *sg)
+void amdgpu_amdkfd_gpuvm_put_sg_table(struct sg_table *sg)
{
sg_free_table(sg);
kfree(sg);
-
- unpin_bo_wo_map(mem);
}
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
--
2.29.2
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH] drm/amdgpu: Limit the maximum size of contiguous VRAM that can be encapsulated by an instance of DRM memory node
[Why]
Enable 1:1 mapping between VRAM of a DRM node and a scatterlist node
[How]
Ensure construction of DRM node to not exceed specified limit
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 20747d8b1135..21d18efca277 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -476,6 +476,9 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager
*man,
for (i = 0; pages_left >= pages_per_node; ++i) {
unsigned long pages = rounddown_pow_of_two(pages_left);
+ /* Limit maximum size to 2GB due to SG table limitations */
+ pages = min(pages, (2UL << (30 - PAGE_SHIFT)));
+
r = drm_mm_insert_node_in_range(mm, &nodes[i], pages,
pages_per_node, 0,
place->fpfn, lpfn,
--
2.29.2
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH] drm/amd/amdgpu: Import amdgpu_amdkfd_gfx_v9 header file into implementation
[Why]
Prevent compiler warnings when building amdgpu kernel module
[How]
Add the include directive for amdgpu_amdkfd_gfx_v9 header file
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index b75bf0bb05ae..c70d8cadaf18 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -37,6 +37,7 @@
#include "soc15.h"
#include "soc15d.h"
#include "gfx_v9_0.h"
+#include "amdgpu_amdkfd_gfx_v9.h"
enum hqd_dequeue_request_type {
NO_ACTION = 0,
--
2.17.1
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH] drm/amd/amdgpu: Enable arcturus devices to access the method kgd_gfx_v9_get_cu_occupancy that is already defined
[Why]
Allow user to know number of compute units (CU) that are in use at any
given moment.
[How]
Remove the keyword static for the method kgd_gfx_v9_get_cu_occupancy
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 2 ++
3 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
index 1afa8f122e7d..604757a1e440 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
@@ -304,4 +304,5 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = {
kgd_gfx_v9_get_atc_vmid_pasid_mapping_info,
.set_vm_context_page_table_base =
kgd_gfx_v9_set_vm_context_page_table_base,
+ .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy
};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index e0d5110701bd..b75bf0bb05ae 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -799,7 +799,7 @@ static void get_wave_count(struct amdgpu_device *adev, int
queue_idx,
*
* Reading registers referenced above involves programming GRBM appropriately
*/
-static void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
+void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
int *pasid_wave_cnt, int *max_waves_per_cu)
{
int qidx;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
index f098e88d3a0d..de9eca55b0ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
@@ -63,3 +63,5 @@ bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct
kgd_dev *kgd,
void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd,
uint32_t vmid, uint64_t page_table_base);
+void kgd_gfx_v9_get_cu_occupancy(struct kgd_dev *kgd, int pasid,
+ int *pasid_wave_cnt, int *max_waves_per_cu);
--
2.17.1
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 2/4] drm/amd/amdgpu: Prepare implementation to support reporting of CU usage
[Why]
Allow user to know number of compute units (CU) that are in use at any
given moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 5 +++--
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h | 3 +++
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 346d8288f6ab..6959aebae6d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -49,6 +49,7 @@
#include "amdgpu_ras.h"
#include "gfx_v9_4.h"
+#include "gfx_v9_0.h"
#include "asic_reg/pwr/pwr_10_0_offset.h"
#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
@@ -788,7 +789,6 @@ static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device
*adev);
static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
struct amdgpu_cu_info *cu_info);
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance);
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
@@ -2397,7 +2397,8 @@ static void gfx_v9_0_tiling_mode_table_init(struct
amdgpu_device *adev)
/* TODO */
}
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance)
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
+ u32 instance)
{
u32 data;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
index 407472031daa..dfe8d4841f58 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
@@ -26,4 +26,7 @@
extern const struct amdgpu_ip_block_version gfx_v9_0_ip_block;
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
+ u32 instance);
+
#endif
--
2.27.0
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 4/4] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h| 25 +
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 71 +++-
2 files changed, 94 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 56f92cfff591..b7be5c5751b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -705,6 +705,31 @@ struct kfd_process_device {
struct kobject *kobj_stats;
unsigned int doorbell_index;
+
+ /*
+* @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+* that is associated with device encoded by "this" struct instance. The
+* value reflects CU usage by all of the waves launched by this process
+* on this device. A very important property of occupancy parameter is
+* that its value is a snapshot of current use.
+*
+* Following is to be noted regarding how this parameter is reported:
+*
+* The number of waves that a CU can launch is limited by couple of
+* parameters. These are encoded by struct amdgpu_cu_info instance
+* that is part of every device definition. For GFX9 devices this
+* translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+* do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+* when they do use scratch memory. This could change for future
+* devices and therefore this example should be considered as a guide.
+*
+* All CU's of a device are available for the process. This may not be
true
+* under certain conditions - e.g. CU masking.
+*
+* Finally number of CU's that are occupied by a process is affected
by both
+* number of CU's a device has along with number of other competing
processes
+*/
+ struct attribute attr_cu_occupancy;
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 17d909c86f50..2807e1c4d59b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct
*work)
}
}
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute
units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing
collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+ int cu_cnt;
+ int wave_cnt;
+ int max_waves_per_cu;
+ struct kfd_dev *dev = NULL;
+ struct kfd_process *proc = NULL;
+ struct kfd_process_device *pdd = NULL;
+
+ pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+ dev = pdd->dev;
+ if (dev->kfd2kgd->get_cu_occupancy == NULL)
+ return -EINVAL;
+
+ cu_cnt = 0;
+ proc = pdd->process;
+ if (pdd->qpd.queue_count == 0) {
+ pr_debug("Gpu-Id: %d has no active queues for process %d\n",
+dev->id, proc->pasid);
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+ }
+
+ /* Collect wave count from device if it supports */
+ wave_cnt = 0;
+ max_waves_per_cu = 0;
+ dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
+ &max_waves_per_cu);
+
+ /* Translate wave count to number of compute units */
+ cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+}
+
static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
char *buffer)
{
@@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
return 0;
}
+
static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
struct attribute *attr, char *buffer)
{
@@ -359,8 +406,13 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
PAGE_SIZE,
[PATCH 3/4] drm/amd/amdgpu: Define and implement a function that collects number of waves that are in flight.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 176 +-
.../gpu/drm/amd/include/kgd_kfd_interface.h | 12 ++
2 files changed, 187 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index e6aede725197..14a0f277e53e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -38,7 +38,7 @@
#include "soc15d.h"
#include "mmhub_v1_0.h"
#include "gfxhub_v1_0.h"
-
+#include "gfx_v9_0.h"
enum hqd_dequeue_request_type {
NO_ACTION = 0,
@@ -706,6 +706,179 @@ void kgd_gfx_v9_set_vm_context_page_table_base(struct
kgd_dev *kgd,
gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base);
}
+static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+ mutex_lock(&adev->srbm_mutex);
+ mutex_lock(&adev->grbm_idx_mutex);
+
+}
+
+static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+ mutex_unlock(&adev->grbm_idx_mutex);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+/**
+ * @get_wave_count: Read device registers to get number of waves in flight for
+ * a particular queue. The method also returns the VMID associated with the
+ * queue.
+ *
+ * @adev: Handle of device whose registers are to be read
+ * @queue_idx: Index of queue in the queue-map bit-field
+ * @wave_cnt: Output parameter updated with number of waves in flight
+ * @vmid: Output parameter updated with VMID of queue whose wave count
+ * is being collected
+ */
+static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
+ int *wave_cnt, int *vmid)
+{
+ int pipe_idx;
+ int queue_slot;
+ unsigned int reg_val;
+
+ /*
+* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID
+* parameters to read out waves in flight. Get VMID if there are
+* non-zero waves in flight.
+*/
+ *vmid = 0xFF;
+ *wave_cnt = 0;
+ pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
+ queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
+ soc15_grbm_select(adev, 1, pipe_idx, queue_slot, 0);
+ reg_val = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CSQ_WF_ACTIVE_COUNT_0) +
+queue_slot);
+ *wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
+ if (*wave_cnt != 0)
+ *vmid = (RREG32_SOC15(GC, 0, mmCP_HQD_VMID) &
+CP_HQD_VMID__VMID_MASK) >> CP_HQD_VMID__VMID__SHIFT;
+}
+
+/**
+ * @kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each
+ * shader engine and aggregates the number of waves that are in flight for the
+ * process whose pasid is provided as a parameter. The process could have ZERO
+ * or more queues running and submitting waves to compute units.
+ *
+ * @kgd: Handle of device from which to get number of waves in flight
+ * @pasid: Identifies the process for which this query call is invoked
+ * @wave_cnt: Output parameter updated with number of waves in flight that
+ * belong to process with given pasid
+ * @max_waves_per_cu: Output parameter updated with maximum number of waves
+ * possible per Compute Unit
+ *
+ * @note: It's possible that the device has too many queues (oversubscription)
+ * in which case a VMID could be remapped to a different PASID. This could lead
+ * to an iaccurate wave count. Following is a high-level sequence:
+ *Time T1: vmid = getVmid(); vmid is associated with Pasid P1
+ *Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
+ * In the sequence above wave count obtained from time T1 will be incorrectly
+ * lost or added to total wave count.
+ *
+ * The registers that provide the waves in flight are:
+ *
+ * SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a
+ * queue is slotted, OFF if there is no queue. A process could have ZERO or
+ * more queues slotted and submitting waves to be run on compute units. Even
+ * when there is a queue it is possible there could be zero wave fronts, this
+ * can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem
+ * command
+ *
+ * For each bit that is ON from above:
+ *
+ *Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the
+ *number of waves that are in flight for the queue at specified index. The
+ *index ranges from 0 to 7.
+ *
+ *If non-zero waves are in flight, read CP_HQD_VMID register to obtain VMID
+ *of the wave(s).
+ *
+ *Determine if VMID from above step maps to pasid provided as paramet
[PATCH 2/4] drm/amd/amdgpu: Prepare implementation to support reporting of CU usage
[Why]
Allow user to know number of compute units (CU) that are in use at any
given moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 9 +
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h | 3 +++
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 346d8288f6ab..b1ab9fd02db7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -49,6 +49,7 @@
#include "amdgpu_ras.h"
#include "gfx_v9_4.h"
+#include "gfx_v9_0.h"
#include "asic_reg/pwr/pwr_10_0_offset.h"
#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
@@ -786,9 +787,8 @@ static void gfx_v9_0_set_irq_funcs(struct amdgpu_device
*adev);
static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
- struct amdgpu_cu_info *cu_info);
+ struct amdgpu_cu_info *cu_info);
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance);
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
@@ -2397,7 +2397,8 @@ static void gfx_v9_0_tiling_mode_table_init(struct
amdgpu_device *adev)
/* TODO */
}
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance)
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
+ u32 instance)
{
u32 data;
@@ -6924,7 +6925,7 @@ static u32 gfx_v9_0_get_cu_active_bitmap(struct
amdgpu_device *adev)
}
static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
-struct amdgpu_cu_info *cu_info)
+ struct amdgpu_cu_info *cu_info)
{
int i, j, k, counter, active_cu_number = 0;
u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
index 407472031daa..dfe8d4841f58 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
@@ -26,4 +26,7 @@
extern const struct amdgpu_ip_block_version gfx_v9_0_ip_block;
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
+ u32 instance);
+
#endif
--
2.27.0
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 1/4] drm/amd/amdgpu: Clean up header file of symbols that are defined to be static
[Why] Header file exports functions get_gpu_clock_counter(), get_cu_info() and select_se_sh() that are defined to be static Signed-off-by: Ramesh Errabolu --- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h | 5 - 1 file changed, 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h index fa5a3fbaf6ab..407472031daa 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h @@ -26,9 +26,4 @@ extern const struct amdgpu_ip_block_version gfx_v9_0_ip_block; -void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num); - -uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); -int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, struct amdgpu_cu_info *cu_info); - #endif -- 2.27.0 ___ amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h| 25 +
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 68 +++-
2 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 56f92cfff591..3df2b9936458 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -705,6 +705,31 @@ struct kfd_process_device {
struct kobject *kobj_stats;
unsigned int doorbell_index;
+
+ /*
+* @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+* that is associated with device encoded by "this" struct instance. The
+* value reflects CU usage by all of the waves launched by this process
+* on this device. A very important property of occupancy parameter is
+* that its value is a a snapshot of current use.
+*
+* Following is to be noted regarding how this parameter is reported:
+*
+* The number of waves that a CU can launch is limited by couple of
+* parameters. These are encoded by struct amdgpu_cu_info instance
+* that is part of every device definition. For GFX9 devices this
+* translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+* do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+* when they use. This could change for future devices and therefore
+* this example should be considered as a guide.
+*
+* All CU's of a device are available for the process. This may not be
true
+* under certain conditions - e.g. CU masking.
+*
+* Finally number of CU's that are occupied by a process is affected
by both
+* number of CU's a device has along with number of other competing
processes
+*/
+ struct attribute attr_cu_occupancy;
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 17d909c86f50..26b716b5eb23 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct
*work)
}
}
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute
units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing
collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+ int cu_cnt;
+ int wave_cnt;
+ int max_waves_per_cu;
+ struct kfd_dev *dev = NULL;
+ struct kfd_process *proc = NULL;
+ struct kfd_process_device *pdd = NULL;
+
+ pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+ dev = pdd->dev;
+ if (dev->kfd2kgd->get_cu_occupancy == NULL)
+ return -EINVAL;
+
+ cu_cnt = 0;
+ proc = pdd->process;
+ if (pdd->qpd.queue_count == 0) {
+ pr_debug("Gpu-Id: %d has no active queues for process %d\n",
+dev->id, proc->pasid);
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+ }
+
+ /* Collect wave count from device if it supports */
+ wave_cnt = 0;
+ max_waves_per_cu = 0;
+ dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
+ &max_waves_per_cu);
+
+ /* Translate wave count to number of compute units */
+ cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+}
+
static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
char *buffer)
{
@@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
return 0;
}
+
static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
struct attribute *attr, char *buffer)
{
@@ -359,6 +406,10 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
PAGE_SIZE,
"%llu\n",
[PATCH 2/3] drm/amd/amdgpu: Define and implement a function that collects number of waves that are in flight.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 176 +-
.../gpu/drm/amd/include/kgd_kfd_interface.h | 12 ++
2 files changed, 187 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index e6aede725197..87d4c8855805 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -38,7 +38,7 @@
#include "soc15d.h"
#include "mmhub_v1_0.h"
#include "gfxhub_v1_0.h"
-
+#include "gfx_v9_0.h"
enum hqd_dequeue_request_type {
NO_ACTION = 0,
@@ -706,6 +706,179 @@ void kgd_gfx_v9_set_vm_context_page_table_base(struct
kgd_dev *kgd,
gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base);
}
+static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+ mutex_lock(&adev->srbm_mutex);
+ mutex_lock(&adev->grbm_idx_mutex);
+
+}
+
+static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+ mutex_unlock(&adev->grbm_idx_mutex);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+/**
+ * @get_wave_count: Read device registers to get number of waves in flight for
+ * a particulare queue. The method also returns the VMID associated with the
+ * queue.
+ *
+ * @adev: Handle of device whose registers are to be read
+ * @queue_idx: Index of queue in the queue-map bit-field
+ * @wave_cnt: Output parameter updated with number of waves in flight
+ * @vmid: Output parameter updated with VMID of queue whose wave count
+ * is being collected
+ */
+static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
+ int *wave_cnt, int *vmid)
+{
+ int pipe_idx;
+ int queue_slot;
+ unsigned int reg_val;
+
+ /*
+* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID
+* parameters to read out waves in flight. Get VMID if there are
+* non-zero waves in flight.
+*/
+ *vmid = 0xFF;
+ *wave_cnt = 0;
+ pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
+ queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
+ soc15_grbm_select(adev, 1, pipe_idx, queue_slot, 0);
+ reg_val = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CSQ_WF_ACTIVE_COUNT_0) +
+queue_slot);
+ *wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
+ if (*wave_cnt != 0)
+ *vmid = (RREG32_SOC15(GC, 0, mmCP_HQD_VMID) &
+CP_HQD_VMID__VMID_MASK) >> CP_HQD_VMID__VMID__SHIFT;
+}
+
+/**
+ * @kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each
+ * shader engine and aggregates the number of waves that are in fight for the
+ * process whose pasid is provided as a parameter. The process could have ZERO
+ * or more queues running and submitting waves to compute units.
+ *
+ * @kgd: Handle of device from which to get number of waves in flight
+ * @pasid: Identifies the process for which this query call is invoked
+ * @wave_cnt: Output parameter updated with number of waves in flight that
+ * belong to process with given pasid
+ * @max_waves_per_cu: Output parameter updated with maximum number of waves
+ * possible per Compute Unit
+ *
+ * @note: It's possible that the device has too many queues (oversubscription)
+ * in which case a VMID could be remapped to a different PASID. This could lead
+ * to in accurate wave count. Following is a high-level sequence:
+ *Time T1: vmid = getVmid(); vmid is associated with Pasid P1
+ *Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
+ * In the sequence above wave count obtained from time T1 will be incorrectly
+ * lost or added to total wave count.
+ *
+ * The registers that provide the waves in flight are:
+ *
+ * SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a
+ * queue is slotted, OFF if there is no queue. A process could have ZERO or
+ * more queues slotted and submitting waves to be run on compute units. Even
+ * when there is a queue it is possible there could be zero wave fronts, this
+ * can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem
+ * command
+ *
+ * For each bit that is ON from above:
+ *
+ *Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the
+ *number of waves that are in flight for the queue at specified index. The
+ *index ranges from 0 to 7.
+ *
+ *If non-zero waves are in fligth, read CP_HQD_VMID register to obtain VMID
+ *of the wave(s).
+ *
+ *Determine if VMID from above step maps to pasid provided as paramet
[PATCH 1/3] drm/amd/amdgpu: Prepare implementation to support reporting of CU usage
[Why]
Allow user to know number of compute units (CU) that are in use at any
given moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 11 ++-
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h | 5 +
2 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 346d8288f6ab..75a17a4007ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -49,6 +49,7 @@
#include "amdgpu_ras.h"
#include "gfx_v9_4.h"
+#include "gfx_v9_0.h"
#include "asic_reg/pwr/pwr_10_0_offset.h"
#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
@@ -785,10 +786,9 @@ static void gfx_v9_0_set_ring_funcs(struct amdgpu_device
*adev);
static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
-static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
- struct amdgpu_cu_info *cu_info);
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance);
+static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
+ struct amdgpu_cu_info *cu_info);
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
@@ -2397,7 +2397,8 @@ static void gfx_v9_0_tiling_mode_table_init(struct
amdgpu_device *adev)
/* TODO */
}
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance)
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
+ u32 instance)
{
u32 data;
@@ -6924,7 +6925,7 @@ static u32 gfx_v9_0_get_cu_active_bitmap(struct
amdgpu_device *adev)
}
static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
-struct amdgpu_cu_info *cu_info)
+struct amdgpu_cu_info *cu_info)
{
int i, j, k, counter, active_cu_number = 0;
u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
index fa5a3fbaf6ab..37eba971acb1 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
@@ -26,9 +26,6 @@
extern const struct amdgpu_ip_block_version gfx_v9_0_ip_block;
-void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num);
-
-uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
-int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, struct amdgpu_cu_info
*cu_info);
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
u32 instance);
#endif
--
2.27.0
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h| 28 ++-
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 101 +++
2 files changed, 114 insertions(+), 15 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 023629f28495..2ce03586d0dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -631,7 +631,7 @@ enum kfd_pdd_bound {
PDD_BOUND_SUSPENDED,
};
-#define MAX_SYSFS_FILENAME_LEN 11
+#define MAX_SYSFS_FILENAME_LEN 32
/*
* SDMA counter runs at 100MHz frequency.
@@ -692,6 +692,32 @@ struct kfd_process_device {
uint64_t sdma_past_activity_counter;
struct attribute attr_sdma;
char sdma_filename[MAX_SYSFS_FILENAME_LEN];
+
+ /*
+* @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+* that is associated with device encoded by "this" struct instance. The
+* value reflects CU usage by all of the waves launched by this process
+* on this device. A very important property of occupancy parameter is
+* that its value is a a snapshot of current use.
+*
+* Following is to be noted regarding how this parameter is reported:
+*
+* The number of waves that a CU can launch is limited by couple of
+* parameters. These are encoded by struct amdgpu_cu_info instance
+* that is part of every device definition. For GFX9 devices this
+* translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+* do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+* when they use. This could change for future devices and therefore
+* this example should be considered as a guide.
+*
+* All CU's of a device are available for the process. This may not be
true
+* under certain conditions - e.g. CU masking.
+*
+* Finally number of CU's that are occupied by a process is affected
by both
+* number of CU's a device has along with number of other competing
processes
+*/
+ struct attribute attr_cu_occupancy;
+ char cu_occupancy_filename[MAX_SYSFS_FILENAME_LEN];
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index a0e12a79ab7d..3baffbc828b2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,63 @@ static void kfd_sdma_activity_worker(struct work_struct
*work)
}
}
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute
units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing
collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+ int cu_cnt;
+ int wave_cnt;
+ int max_waves_per_cu;
+ struct kfd_dev *dev = NULL;
+ struct kfd_process *proc = NULL;
+ struct amdgpu_device *adev = NULL;
+ struct kfd_process_device *pdd = NULL;
+
+ /* Acquire handle of Process-Device-Data associated with attribute */
+ pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+
+ /*
+* Acquire handle of Gpu Device associated with attribute. Determine
+* if ability to collect CU occupancy is defined for this device. If
+* so acquire the handle of process which encapsulates the PASID of
+* process. It is possible that the process might have zero work on
+* device. This is determined by checking if process has any queues
+* @note: Ability to collect is defined only for Gfx9 devices
+*/
+ cu_cnt = 0;
+ dev = pdd->dev;
+ proc = pdd->process;
+ if (pdd->qpd.queue_count == 0) {
+ pr_info("%s: Gpu-Id: %d has no active queues for process %d\n",
+ __func__, dev->id, proc->pasid);
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+ }
+
+ /* Collect wave count from device by reading relevant registers */
+ wave_cnt = 0;
+ if (dev->kfd2kgd->get_cu_occupancy != NULL)
+
[PATCH 1/3] drm/amd/amdgpu: Prepare implementation to support reporting of CU usage
[Why]
Allow user to know number of compute units (CU) that are in use at any
given moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 14 ++
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h | 3 +--
2 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index d898c9ff3526..b31879be2c05 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -49,6 +49,7 @@
#include "amdgpu_ras.h"
#include "gfx_v9_4.h"
+#include "gfx_v9_0.h"
#include "asic_reg/pwr/pwr_10_0_offset.h"
#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
@@ -785,10 +786,6 @@ static void gfx_v9_0_set_ring_funcs(struct amdgpu_device
*adev);
static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
-static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
- struct amdgpu_cu_info *cu_info);
-static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance);
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
@@ -2402,7 +2399,8 @@ static void gfx_v9_0_tiling_mode_table_init(struct
amdgpu_device *adev)
/* TODO */
}
-static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32
sh_num, u32 instance)
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
+ u32 instance)
{
u32 data;
@@ -4117,7 +4115,7 @@ static uint64_t gfx_v9_0_kiq_read_clock(struct
amdgpu_device *adev)
return ~0;
}
-static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
{
uint64_t clock;
@@ -6928,8 +6926,8 @@ static u32 gfx_v9_0_get_cu_active_bitmap(struct
amdgpu_device *adev)
return (~data) & mask;
}
-static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
-struct amdgpu_cu_info *cu_info)
+int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
+struct amdgpu_cu_info *cu_info)
{
int i, j, k, counter, active_cu_number = 0;
u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
index fa5a3fbaf6ab..a3d73667e60a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
@@ -26,9 +26,8 @@
extern const struct amdgpu_ip_block_version gfx_v9_0_ip_block;
-void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num);
-
uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, struct amdgpu_cu_info
*cu_info);
+void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
u32 instance);
#endif
--
2.27.0
___
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[PATCH 2/3] drm/amd/amdgpu: Define and implement a function that collects number of waves that are in flight.
[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.
[How]
Read registers of SQ that give number of waves that are in flight
of various queues. Use this information to determine number of CU's
in use.
Signed-off-by: Ramesh Errabolu
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 206 ++
.../gpu/drm/amd/include/kgd_kfd_interface.h | 11 +
2 files changed, 217 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index e6aede725197..2f8c8140734e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -38,7 +38,9 @@
#include "soc15d.h"
#include "mmhub_v1_0.h"
#include "gfxhub_v1_0.h"
+#include "gfx_v9_0.h"
+struct kfd_dev;
enum hqd_dequeue_request_type {
NO_ACTION = 0,
@@ -706,6 +708,209 @@ void kgd_gfx_v9_set_vm_context_page_table_base(struct
kgd_dev *kgd,
gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base);
}
+static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+ mutex_lock(&adev->srbm_mutex);
+ mutex_lock(&adev->grbm_idx_mutex);
+
+}
+
+static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+ mutex_unlock(&adev->grbm_idx_mutex);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+/**
+ * @get_wave_count: Read device registers to get number of waves in flight for
+ * a particulare queue. The method also returns the VMID associated with the
+ * queue.
+ *
+ * @adev: Handle of device whose registers are to be read
+ *
+ * @queue_idx: Index of queue in the queue-map bit-field
+ *
+ * @wave_cnt: Output parameter updated with number of waves in flight
+ *
+ * @vmid: Output parameter updated with VMID of queue whose wave count
+ * is being collected
+ */
+static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
+ int *wave_cnt, int *vmid)
+{
+ int pipe_idx;
+ int queue_slot;
+ unsigned int reg_val;
+
+ /*
+* By policy queues at slots 0 and 1 are reserved for non-compute
+* queues i.e. those managed for graphic functions.
+*/
+ if ((queue_idx % adev->gfx.mec.num_queue_per_pipe) < 2)
+ return;
+
+ /*
+* Queue belongs to a compute workload. Determine the PIPE index
+* associated wit queue and program GRBM accordingly:
+* MEID = 1, PIPEID = pipe_idx, QUEUEID = queue_idx, VMID = 0
+*/
+ pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
+ queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
+ soc15_grbm_select(adev, 1, pipe_idx, queue_slot, 0);
+
+ /*
+* Read from register number of waves in flight. If non-zero get the
+* VMID associated with queue
+*/
+ reg_val = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CSQ_WF_ACTIVE_COUNT_0) +
+queue_slot);
+ *wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
+ if (*wave_cnt != 0)
+ *vmid = (RREG32_SOC15(GC, 0, mmCP_HQD_VMID) &
+CP_HQD_VMID__VMID_MASK) >> CP_HQD_VMID__VMID__SHIFT;
+}
+
+/**
+ * @kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each
+ * shader engine and aggregates the number of waves that are in fight for the
+ * process whose pasid is provided as a parameter. The process could have ZERO
+ * or more queues running and submitting waves to compute units.
+ *
+ * @note: It's possible that the device has too many queues (oversubscription)
+ * in which case a VMID could be remapped to a different PASID. This could lead
+ * to in accurate wave count. Following is a high-level sequence:
+ *Time T1: vmid = getVmid(); vmid is associated with Pasid P1
+ *Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
+ * In the sequence above wave count obtained from time T1 will be incorrectly
+ * lost or added to total wave count.
+ *
+ * @kgd: Handle of device from which to get number of waves in flight
+ *
+ * @pasid: Identifies the process for which this query call is invoked
+ *
+ * @wave_cnt: Output parameter updated with number of waves in flight that
+ * belong to process with given pasid
+ *
+ * The registers that provide the waves in flight are:
+ *
+ * SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. At any moment there
+ * can be a max of 32 queues that could submit wave fronts to be run by
compute
+ * units. The bit is ON if a queue is slotted, OFF if there is no queue. The
+ * process could have ZERO or more queues slotted and submitting waves to be
+ * run compute units. Even when there is a queue it is possible there could
+ * be zero wave fronts, this can happen when queue is waiting on top-of-pipe
+ * events - e.g. waitRegMem command
+ *
+ * For each bit
