[PATCH] amdgpu: fix asic initialization for virtualized environments
On Jun 11, 2016 12:09 AM, "Andres Rodriguez"
wrote:
>
> When executing in a PCI passthrough based virtuzliation environemnt, the
> hypervisor will usually attempt to send a PCIe bus reset signal to the
> ASIC when the VM reboots. In this scenario, the card is not correctly
> initialized, but we still consider it to be posted. Therefore, in a
> passthrough based environemnt we should always post the card to guarantee
> it is in a good state for driver initialization.
>
> However, if we are operating in SR-IOV mode it is up to the GIM driver
> to manage the asic state, therefore we should not post the card (and
> shouldn't be able to do it either).
>
> Signed-off-by: Andres Rodriguez
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h| 7 +++
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 16 +++-
> drivers/gpu/drm/amd/amdgpu/cik.c | 7 +++
> drivers/gpu/drm/amd/amdgpu/vi.c| 15 +++
> 4 files changed, 44 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 26fe670..fe71dea 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -1835,6 +1835,8 @@ struct amdgpu_asic_funcs {
> /* MM block clocks */
> int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32
dclk);
> int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32
ecclk);
> + /* query virtual capabilities */
> + u32 (*get_virtual_caps)(struct amdgpu_device *adev);
> };
>
> /*
> @@ -1932,8 +1934,12 @@ void amdgpu_cgs_destroy_device(struct cgs_device
*cgs_device);
>
>
> /* GPU virtualization */
> +#define AMDGPU_VIRT_CAPS_SRIOV_EN (1 << 0)
> +#define AMDGPU_VIRT_CAPS_IS_VF (1 << 1)
> struct amdgpu_virtualization {
> bool supports_sr_iov;
> + bool is_virtual;
> + u32 caps;
> };
>
> /*
> @@ -2226,6 +2232,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
> #define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
> #define amdgpu_asic_set_uvd_clocks(adev, v, d)
(adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
> #define amdgpu_asic_set_vce_clocks(adev, ev, ec)
(adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
> +#define amdgpu_asic_get_virtual_caps(adev)
((adev)->asic_funcs->get_virtual_caps((adev)))
> #define amdgpu_asic_get_gpu_clock_counter(adev)
(adev)->asic_funcs->get_gpu_clock_counter((adev))
> #define amdgpu_asic_read_disabled_bios(adev)
(adev)->asic_funcs->read_disabled_bios((adev))
> #define amdgpu_asic_read_bios_from_rom(adev, b, l)
(adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index b494212..629e4e3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -1392,6 +1392,14 @@ static int amdgpu_resume(struct amdgpu_device
*adev)
> return 0;
> }
>
> +static bool amdgpu_device_is_virtual(void)
> +{
> +#ifdef CONFIG_X86
> + return boot_cpu_has(X86_FEATURE_HYPERVISOR);
> +#else
> + return false
Missing semicolon here...
> +#endif
> +}
>
> /**
> * amdgpu_device_has_dal_support - check if dal is supported
> @@ -1560,8 +1568,14 @@ int amdgpu_device_init(struct amdgpu_device *adev,
> adev->virtualization.supports_sr_iov =
> amdgpu_atombios_has_gpu_virtualization_table(adev);
>
> + /* Check if we are executing in a virtualized environment */
> + adev->virtualization.is_virtual = amdgpu_device_is_virtual();
> + adev->virtualization.caps = amdgpu_asic_get_virtual_caps(adev);
> +
> /* Post card if necessary */
> - if (!amdgpu_card_posted(adev)) {
> + if (!amdgpu_card_posted(adev) ||
> + (adev->virtualization.is_virtual &&
> +!adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN)) {
> if (!adev->bios) {
> dev_err(adev->dev, "Card not posted and no BIOS -
ignoring\n");
> return -EINVAL;
> diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c
b/drivers/gpu/drm/amd/amdgpu/cik.c
> index 40f4fda..907bb28 100644
> --- a/drivers/gpu/drm/amd/amdgpu/cik.c
> +++ b/drivers/gpu/drm/amd/amdgpu/cik.c
> @@ -963,6 +963,12 @@ static bool cik_read_bios_from_rom(struct
amdgpu_device *adev,
> return true;
> }
>
> +static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
> +{
> + /* CIK does not support SR-IOV */
> + return 0;
> +}
> +
> static const struct amdgpu_allowed_register_entry
cik_allowed_read_registers[] = {
> {mmGRBM_STATUS, false},
> {mmGB_ADDR_CONFIG, false},
> @@ -2176,6 +2182,7 @@ static const struct amdgpu_asic_funcs
cik_asic_funcs =
> .get_xclk = &cik_get_xclk,
> .set_uvd_clocks = &cik_set_uvd_clocks,
> .set_vce_clocks = &cik_set_vce_clocks,
> + .get_virtual_caps
[PATCH] amdgpu: fix asic initialization for virtualized environments
When executing in a PCI passthrough based virtuzliation environemnt, the
hypervisor will usually attempt to send a PCIe bus reset signal to the
ASIC when the VM reboots. In this scenario, the card is not correctly
initialized, but we still consider it to be posted. Therefore, in a
passthrough based environemnt we should always post the card to guarantee
it is in a good state for driver initialization.
However, if we are operating in SR-IOV mode it is up to the GIM driver
to manage the asic state, therefore we should not post the card (and
shouldn't be able to do it either).
Signed-off-by: Andres Rodriguez
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h| 7 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 16 +++-
drivers/gpu/drm/amd/amdgpu/cik.c | 7 +++
drivers/gpu/drm/amd/amdgpu/vi.c| 15 +++
4 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 26fe670..fe71dea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1835,6 +1835,8 @@ struct amdgpu_asic_funcs {
/* MM block clocks */
int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
+ /* query virtual capabilities */
+ u32 (*get_virtual_caps)(struct amdgpu_device *adev);
};
/*
@@ -1932,8 +1934,12 @@ void amdgpu_cgs_destroy_device(struct cgs_device
*cgs_device);
/* GPU virtualization */
+#define AMDGPU_VIRT_CAPS_SRIOV_EN (1 << 0)
+#define AMDGPU_VIRT_CAPS_IS_VF (1 << 1)
struct amdgpu_virtualization {
bool supports_sr_iov;
+ bool is_virtual;
+ u32 caps;
};
/*
@@ -2226,6 +2232,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
#define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
#define amdgpu_asic_set_uvd_clocks(adev, v, d)
(adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
#define amdgpu_asic_set_vce_clocks(adev, ev, ec)
(adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
+#define amdgpu_asic_get_virtual_caps(adev)
((adev)->asic_funcs->get_virtual_caps((adev)))
#define amdgpu_asic_get_gpu_clock_counter(adev)
(adev)->asic_funcs->get_gpu_clock_counter((adev))
#define amdgpu_asic_read_disabled_bios(adev)
(adev)->asic_funcs->read_disabled_bios((adev))
#define amdgpu_asic_read_bios_from_rom(adev, b, l)
(adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index b494212..629e4e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1392,6 +1392,14 @@ static int amdgpu_resume(struct amdgpu_device *adev)
return 0;
}
+static bool amdgpu_device_is_virtual(void)
+{
+#ifdef CONFIG_X86
+ return boot_cpu_has(X86_FEATURE_HYPERVISOR);
+#else
+ return false
+#endif
+}
/**
* amdgpu_device_has_dal_support - check if dal is supported
@@ -1560,8 +1568,14 @@ int amdgpu_device_init(struct amdgpu_device *adev,
adev->virtualization.supports_sr_iov =
amdgpu_atombios_has_gpu_virtualization_table(adev);
+ /* Check if we are executing in a virtualized environment */
+ adev->virtualization.is_virtual = amdgpu_device_is_virtual();
+ adev->virtualization.caps = amdgpu_asic_get_virtual_caps(adev);
+
/* Post card if necessary */
- if (!amdgpu_card_posted(adev)) {
+ if (!amdgpu_card_posted(adev) ||
+ (adev->virtualization.is_virtual &&
+!adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN)) {
if (!adev->bios) {
dev_err(adev->dev, "Card not posted and no BIOS -
ignoring\n");
return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 40f4fda..907bb28 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -963,6 +963,12 @@ static bool cik_read_bios_from_rom(struct amdgpu_device
*adev,
return true;
}
+static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
+{
+ /* CIK does not support SR-IOV */
+ return 0;
+}
+
static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[]
= {
{mmGRBM_STATUS, false},
{mmGB_ADDR_CONFIG, false},
@@ -2176,6 +2182,7 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
.get_xclk = &cik_get_xclk,
.set_uvd_clocks = &cik_set_uvd_clocks,
.set_vce_clocks = &cik_set_vce_clocks,
+ .get_virtual_caps = &cik_get_virtual_caps,
/* these should be moved to their own ip modules */
.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
.wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu
[PATCH] amdgpu: fix asic initialization for virtualized environments
When executing in a PCI passthrough based virtuzliation environemnt, the
hypervisor will usually attempt to send a PCIe bus reset signal to the
ASIC when the VM reboots. In this scenario, the card is not correctly
initialized, but we still consider it to be posted. Therefore, in a
passthrough based environemnt we should always post the card to guarantee
it is in a good state for driver initialization.
However, if we are operating in SR-IOV mode it is up to the GIM driver
to manage the asic state, therefore we should not post the card (and
shouldn't be able to do it either).
Signed-off-by: Andres Rodriguez
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h| 7 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 16 +++-
drivers/gpu/drm/amd/amdgpu/cik.c | 7 +++
drivers/gpu/drm/amd/amdgpu/vi.c| 15 +++
4 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 26fe670..fe71dea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1835,6 +1835,8 @@ struct amdgpu_asic_funcs {
/* MM block clocks */
int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
+ /* query virtual capabilities */
+ u32 (*get_virtual_caps)(struct amdgpu_device *adev);
};
/*
@@ -1932,8 +1934,12 @@ void amdgpu_cgs_destroy_device(struct cgs_device
*cgs_device);
/* GPU virtualization */
+#define AMDGPU_VIRT_CAPS_SRIOV_EN (1 << 0)
+#define AMDGPU_VIRT_CAPS_IS_VF (1 << 1)
struct amdgpu_virtualization {
bool supports_sr_iov;
+ bool is_virtual;
+ u32 caps;
};
/*
@@ -2226,6 +2232,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
#define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
#define amdgpu_asic_set_uvd_clocks(adev, v, d)
(adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
#define amdgpu_asic_set_vce_clocks(adev, ev, ec)
(adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
+#define amdgpu_asic_get_virtual_caps(adev)
((adev)->asic_funcs->get_virtual_caps((adev)))
#define amdgpu_asic_get_gpu_clock_counter(adev)
(adev)->asic_funcs->get_gpu_clock_counter((adev))
#define amdgpu_asic_read_disabled_bios(adev)
(adev)->asic_funcs->read_disabled_bios((adev))
#define amdgpu_asic_read_bios_from_rom(adev, b, l)
(adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index b494212..629e4e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1392,6 +1392,14 @@ static int amdgpu_resume(struct amdgpu_device *adev)
return 0;
}
+static bool amdgpu_device_is_virtual(void)
+{
+#ifdef CONFIG_X86
+ return boot_cpu_has(X86_FEATURE_HYPERVISOR);
+#else
+ return false
+#endif
+}
/**
* amdgpu_device_has_dal_support - check if dal is supported
@@ -1560,8 +1568,14 @@ int amdgpu_device_init(struct amdgpu_device *adev,
adev->virtualization.supports_sr_iov =
amdgpu_atombios_has_gpu_virtualization_table(adev);
+ /* Check if we are executing in a virtualized environment */
+ adev->virtualization.is_virtual = amdgpu_device_is_virtual();
+ adev->virtualization.caps = amdgpu_asic_get_virtual_caps(adev);
+
/* Post card if necessary */
- if (!amdgpu_card_posted(adev)) {
+ if (!amdgpu_card_posted(adev) ||
+ (adev->virtualization.is_virtual &&
+!adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN)) {
if (!adev->bios) {
dev_err(adev->dev, "Card not posted and no BIOS -
ignoring\n");
return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 40f4fda..907bb28 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -963,6 +963,12 @@ static bool cik_read_bios_from_rom(struct amdgpu_device
*adev,
return true;
}
+static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
+{
+ /* CIK does not support SR-IOV */
+ return 0;
+}
+
static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[]
= {
{mmGRBM_STATUS, false},
{mmGB_ADDR_CONFIG, false},
@@ -2176,6 +2182,7 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
.get_xclk = &cik_get_xclk,
.set_uvd_clocks = &cik_set_uvd_clocks,
.set_vce_clocks = &cik_set_vce_clocks,
+ .get_virtual_caps = &cik_get_virtual_caps,
/* these should be moved to their own ip modules */
.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
.wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu
