From: Klaus Jensen <k.jen...@samsung.com>
Commit 1901b4967c3f ("hw/block/nvme: move msix table and pba to BAR 0")
moved the MSI-X table and PBA to BAR 0 to make room for enabling CMR and
PMR at the same time. As reported by Julien Grall in #2184, this breaks
migration through system hibernation.
Add a machine compatibility parameter and set it on machines pre 6.0 to
enable the old behavior automatically, restoring the hibernation
migration support.
Cc: qemu-sta...@nongnu.org
Resolves: https://gitlab.com/qemu-project/qemu/-/issues/2184
Fixes: 1901b4967c3f ("hw/block/nvme: move msix table and pba to BAR 0")
Reported-by: Julien Grall jul...@xen.org
Tested-by: Julien Grall jul...@xen.org
Reviewed-by: Jesper Wendel Devantier <f...@defmacro.it>
Signed-off-by: Klaus Jensen <k.jen...@samsung.com>
---
hw/core/machine.c | 1 +
hw/nvme/ctrl.c | 53 +++++++++++++++++++++++++++++++++--------------
hw/nvme/nvme.h | 1 +
3 files changed, 39 insertions(+), 16 deletions(-)
diff --git a/hw/core/machine.c b/hw/core/machine.c
index 0e9d646b61e1..37e5d4c8dfd5 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -102,6 +102,7 @@ GlobalProperty hw_compat_5_2[] = {
{ "PIIX4_PM", "smm-compat", "on"},
{ "virtio-blk-device", "report-discard-granularity", "off" },
{ "virtio-net-pci-base", "vectors", "3"},
+ { "nvme", "msix-exclusive-bar", "on"},
};
const size_t hw_compat_5_2_len = G_N_ELEMENTS(hw_compat_5_2);
diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
index efcfd7171066..036b15403a02 100644
--- a/hw/nvme/ctrl.c
+++ b/hw/nvme/ctrl.c
@@ -7810,6 +7810,11 @@ static bool nvme_check_params(NvmeCtrl *n, Error **errp)
}
if (n->pmr.dev) {
+ if (params->msix_exclusive_bar) {
+ error_setg(errp, "not enough BARs available to enable PMR");
+ return false;
+ }
+
if (host_memory_backend_is_mapped(n->pmr.dev)) {
error_setg(errp, "can't use already busy memdev: %s",
object_get_canonical_path_component(OBJECT(n->pmr.dev)));
@@ -8113,24 +8118,38 @@ static bool nvme_init_pci(NvmeCtrl *n, PCIDevice
*pci_dev, Error **errp)
pcie_ari_init(pci_dev, 0x100);
}
- /* add one to max_ioqpairs to account for the admin queue pair */
- bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
- &msix_table_offset, &msix_pba_offset);
-
- memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
- memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
- msix_table_offset);
- memory_region_add_subregion(&n->bar0, 0, &n->iomem);
-
- if (pci_is_vf(pci_dev)) {
- pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
- } else {
+ if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) {
+ bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1, 0, NULL, NULL);
+ memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
+ bar_size);
pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
- PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
+ PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
+ ret = msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, errp);
+ } else {
+ assert(n->params.msix_qsize >= 1);
+
+ /* add one to max_ioqpairs to account for the admin queue pair */
+ bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1,
+ n->params.msix_qsize, &msix_table_offset,
+ &msix_pba_offset);
+
+ memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
+ memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
+ msix_table_offset);
+ memory_region_add_subregion(&n->bar0, 0, &n->iomem);
+
+ if (pci_is_vf(pci_dev)) {
+ pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
+ } else {
+ pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
+ PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
+ }
+
+ ret = msix_init(pci_dev, n->params.msix_qsize,
+ &n->bar0, 0, msix_table_offset,
+ &n->bar0, 0, msix_pba_offset, 0, errp);
}
- ret = msix_init(pci_dev, n->params.msix_qsize,
- &n->bar0, 0, msix_table_offset,
- &n->bar0, 0, msix_pba_offset, 0, errp);
+
if (ret == -ENOTSUP) {
/* report that msix is not supported, but do not error out */
warn_report_err(*errp);
@@ -8434,6 +8453,8 @@ static Property nvme_props[] = {
params.sriov_max_vi_per_vf, 0),
DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl,
params.sriov_max_vq_per_vf, 0),
+ DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar,
+ false),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/hw/nvme/nvme.h b/hw/nvme/nvme.h
index 392c02942682..bed8191bd5fd 100644
--- a/hw/nvme/nvme.h
+++ b/hw/nvme/nvme.h
@@ -536,6 +536,7 @@ typedef struct NvmeParams {
uint16_t sriov_vi_flexible;
uint8_t sriov_max_vq_per_vf;
uint8_t sriov_max_vi_per_vf;
+ bool msix_exclusive_bar;
} NvmeParams;
typedef struct NvmeCtrl {
--
2.44.0
