Re: [PATCH V4 09/43] vfio/container: register container for cpr
On 6/3/25 16:17, Steven Sistare wrote: On 6/3/2025 10:09 AM, Steven Sistare wrote: On 6/3/2025 7:57 AM, Duan, Zhenzhong wrote: -Original Message- From: Steve Sistare Subject: [PATCH V4 09/43] vfio/container: register container for cpr Register a legacy container for cpr-transfer, replacing the generic CPR register call with a more specific legacy container register call. Add a blocker if the kernel does not support VFIO_UPDATE_VADDR or VFIO_UNMAP_ALL. This is mostly boiler plate. The fields to to saved and restored are added in subsequent patches. Signed-off-by: Steve Sistare --- include/hw/vfio/vfio-container.h | 2 ++ include/hw/vfio/vfio-cpr.h | 15 + hw/vfio/container.c | 6 ++-- hw/vfio/cpr-legacy.c | 69 hw/vfio/cpr.c | 5 ++- hw/vfio/meson.build | 1 + 6 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 hw/vfio/cpr-legacy.c diff --git a/include/hw/vfio/vfio-container.h b/include/hw/vfio/vfio-container.h index afc498d..21e5807 100644 --- a/include/hw/vfio/vfio-container.h +++ b/include/hw/vfio/vfio-container.h @@ -10,6 +10,7 @@ #define HW_VFIO_CONTAINER_H #include "hw/vfio/vfio-container-base.h" +#include "hw/vfio/vfio-cpr.h" Now that we have this change, may we remove #include of vfio-cpr.h in hw/vfio/container.c? Maybe this belong to patch8? Yes, thanks. Patch 8 should not add #include of vfio-cpr.h in hw/vfio/container.c However, I see that Cedric has staged these patches in vfio-next. We can make these tweaks in a future patch. It is fine. you can resend. Let's wait first for Michael's feedback on the pci and vfio-pci reset handlers. This is what is blocking me from sending a PR. Thanks, C.
Re: [PATCH V4 09/43] vfio/container: register container for cpr
On 6/3/2025 10:09 AM, Steven Sistare wrote:
On 6/3/2025 7:57 AM, Duan, Zhenzhong wrote:
-Original Message-
From: Steve Sistare
Subject: [PATCH V4 09/43] vfio/container: register container for cpr
Register a legacy container for cpr-transfer, replacing the generic CPR
register call with a more specific legacy container register call. Add a
blocker if the kernel does not support VFIO_UPDATE_VADDR or
VFIO_UNMAP_ALL.
This is mostly boiler plate. The fields to to saved and restored are added
in subsequent patches.
Signed-off-by: Steve Sistare
---
include/hw/vfio/vfio-container.h | 2 ++
include/hw/vfio/vfio-cpr.h | 15 +
hw/vfio/container.c | 6 ++--
hw/vfio/cpr-legacy.c | 69
hw/vfio/cpr.c | 5 ++-
hw/vfio/meson.build | 1 +
6 files changed, 92 insertions(+), 6 deletions(-)
create mode 100644 hw/vfio/cpr-legacy.c
diff --git a/include/hw/vfio/vfio-container.h b/include/hw/vfio/vfio-container.h
index afc498d..21e5807 100644
--- a/include/hw/vfio/vfio-container.h
+++ b/include/hw/vfio/vfio-container.h
@@ -10,6 +10,7 @@
#define HW_VFIO_CONTAINER_H
#include "hw/vfio/vfio-container-base.h"
+#include "hw/vfio/vfio-cpr.h"
Now that we have this change, may we remove #include of vfio-cpr.h in
hw/vfio/container.c?
Maybe this belong to patch8?
Yes, thanks.
Patch 8 should not add #include of vfio-cpr.h in hw/vfio/container.c
However, I see that Cedric has staged these patches in vfio-next.
We can make these tweaks in a future patch.
- Steve
typedef struct VFIOContainer VFIOContainer;
typedef struct VFIODevice VFIODevice;
@@ -29,6 +30,7 @@ typedef struct VFIOContainer {
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
unsigned iommu_type;
QLIST_HEAD(, VFIOGroup) group_list;
+ VFIOContainerCPR cpr;
} VFIOContainer;
OBJECT_DECLARE_SIMPLE_TYPE(VFIOContainer, VFIO_IOMMU_LEGACY);
diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
index 750ea5b..d4e0bd5 100644
--- a/include/hw/vfio/vfio-cpr.h
+++ b/include/hw/vfio/vfio-cpr.h
@@ -9,8 +9,23 @@
#ifndef HW_VFIO_VFIO_CPR_H
#define HW_VFIO_VFIO_CPR_H
+#include "migration/misc.h"
+
+struct VFIOContainer;
struct VFIOContainerBase;
+typedef struct VFIOContainerCPR {
+ Error *blocker;
+} VFIOContainerCPR;
+
+
+bool vfio_legacy_cpr_register_container(struct VFIOContainer *container,
+ Error **errp);
+void vfio_legacy_cpr_unregister_container(struct VFIOContainer *container);
+
+int vfio_cpr_reboot_notifier(NotifierWithReturn *notifier, MigrationEvent *e,
+ Error **errp);
+
bool vfio_cpr_register_container(struct VFIOContainerBase *bcontainer,
Error **errp);
void vfio_cpr_unregister_container(struct VFIOContainerBase *bcontainer);
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 0f948d0..7d2035c 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -643,7 +643,7 @@ static bool vfio_container_connect(VFIOGroup *group,
AddressSpace *as,
new_container = true;
bcontainer = &container->bcontainer;
- if (!vfio_cpr_register_container(bcontainer, errp)) {
+ if (!vfio_legacy_cpr_register_container(container, errp)) {
goto fail;
}
@@ -679,7 +679,7 @@ fail:
vioc->release(bcontainer);
}
if (new_container) {
- vfio_cpr_unregister_container(bcontainer);
+ vfio_legacy_cpr_unregister_container(container);
object_unref(container);
}
if (fd >= 0) {
@@ -720,7 +720,7 @@ static void vfio_container_disconnect(VFIOGroup *group)
VFIOAddressSpace *space = bcontainer->space;
trace_vfio_container_disconnect(container->fd);
- vfio_cpr_unregister_container(bcontainer);
+ vfio_legacy_cpr_unregister_container(container);
close(container->fd);
object_unref(container);
diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c
new file mode 100644
index 000..419b9fb
--- /dev/null
+++ b/hw/vfio/cpr-legacy.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2021-2025 Oracle and/or its affiliates.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-container.h"
+#include "hw/vfio/vfio-cpr.h"
Ditto.
Yes, this #include vfio-cpr.h should be dropped from this patch.
- Steve
Re: [PATCH V4 09/43] vfio/container: register container for cpr
On 6/3/2025 7:57 AM, Duan, Zhenzhong wrote:
-Original Message-
From: Steve Sistare
Subject: [PATCH V4 09/43] vfio/container: register container for cpr
Register a legacy container for cpr-transfer, replacing the generic CPR
register call with a more specific legacy container register call. Add a
blocker if the kernel does not support VFIO_UPDATE_VADDR or
VFIO_UNMAP_ALL.
This is mostly boiler plate. The fields to to saved and restored are added
in subsequent patches.
Signed-off-by: Steve Sistare
---
include/hw/vfio/vfio-container.h | 2 ++
include/hw/vfio/vfio-cpr.h | 15 +
hw/vfio/container.c | 6 ++--
hw/vfio/cpr-legacy.c | 69
hw/vfio/cpr.c| 5 ++-
hw/vfio/meson.build | 1 +
6 files changed, 92 insertions(+), 6 deletions(-)
create mode 100644 hw/vfio/cpr-legacy.c
diff --git a/include/hw/vfio/vfio-container.h b/include/hw/vfio/vfio-container.h
index afc498d..21e5807 100644
--- a/include/hw/vfio/vfio-container.h
+++ b/include/hw/vfio/vfio-container.h
@@ -10,6 +10,7 @@
#define HW_VFIO_CONTAINER_H
#include "hw/vfio/vfio-container-base.h"
+#include "hw/vfio/vfio-cpr.h"
Now that we have this change, may we remove #include of vfio-cpr.h in
hw/vfio/container.c?
Maybe this belong to patch8?
Yes, thanks.
Patch 8 should not add #include of vfio-cpr.h in hw/vfio/container.c
typedef struct VFIOContainer VFIOContainer;
typedef struct VFIODevice VFIODevice;
@@ -29,6 +30,7 @@ typedef struct VFIOContainer {
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
unsigned iommu_type;
QLIST_HEAD(, VFIOGroup) group_list;
+VFIOContainerCPR cpr;
} VFIOContainer;
OBJECT_DECLARE_SIMPLE_TYPE(VFIOContainer, VFIO_IOMMU_LEGACY);
diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
index 750ea5b..d4e0bd5 100644
--- a/include/hw/vfio/vfio-cpr.h
+++ b/include/hw/vfio/vfio-cpr.h
@@ -9,8 +9,23 @@
#ifndef HW_VFIO_VFIO_CPR_H
#define HW_VFIO_VFIO_CPR_H
+#include "migration/misc.h"
+
+struct VFIOContainer;
struct VFIOContainerBase;
+typedef struct VFIOContainerCPR {
+Error *blocker;
+} VFIOContainerCPR;
+
+
+bool vfio_legacy_cpr_register_container(struct VFIOContainer *container,
+Error **errp);
+void vfio_legacy_cpr_unregister_container(struct VFIOContainer *container);
+
+int vfio_cpr_reboot_notifier(NotifierWithReturn *notifier, MigrationEvent *e,
+ Error **errp);
+
bool vfio_cpr_register_container(struct VFIOContainerBase *bcontainer,
Error **errp);
void vfio_cpr_unregister_container(struct VFIOContainerBase *bcontainer);
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 0f948d0..7d2035c 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -643,7 +643,7 @@ static bool vfio_container_connect(VFIOGroup *group,
AddressSpace *as,
new_container = true;
bcontainer = &container->bcontainer;
-if (!vfio_cpr_register_container(bcontainer, errp)) {
+if (!vfio_legacy_cpr_register_container(container, errp)) {
goto fail;
}
@@ -679,7 +679,7 @@ fail:
vioc->release(bcontainer);
}
if (new_container) {
-vfio_cpr_unregister_container(bcontainer);
+vfio_legacy_cpr_unregister_container(container);
object_unref(container);
}
if (fd >= 0) {
@@ -720,7 +720,7 @@ static void vfio_container_disconnect(VFIOGroup *group)
VFIOAddressSpace *space = bcontainer->space;
trace_vfio_container_disconnect(container->fd);
-vfio_cpr_unregister_container(bcontainer);
+vfio_legacy_cpr_unregister_container(container);
close(container->fd);
object_unref(container);
diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c
new file mode 100644
index 000..419b9fb
--- /dev/null
+++ b/hw/vfio/cpr-legacy.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2021-2025 Oracle and/or its affiliates.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-container.h"
+#include "hw/vfio/vfio-cpr.h"
Ditto.
Yes, this #include vfio-cpr.h should be dropped from this patch.
- Steve
RE: [PATCH V4 09/43] vfio/container: register container for cpr
>-Original Message-
>From: Steve Sistare
>Subject: [PATCH V4 09/43] vfio/container: register container for cpr
>
>Register a legacy container for cpr-transfer, replacing the generic CPR
>register call with a more specific legacy container register call. Add a
>blocker if the kernel does not support VFIO_UPDATE_VADDR or
>VFIO_UNMAP_ALL.
>
>This is mostly boiler plate. The fields to to saved and restored are added
>in subsequent patches.
>
>Signed-off-by: Steve Sistare
>---
> include/hw/vfio/vfio-container.h | 2 ++
> include/hw/vfio/vfio-cpr.h | 15 +
> hw/vfio/container.c | 6 ++--
> hw/vfio/cpr-legacy.c | 69
>
> hw/vfio/cpr.c| 5 ++-
> hw/vfio/meson.build | 1 +
> 6 files changed, 92 insertions(+), 6 deletions(-)
> create mode 100644 hw/vfio/cpr-legacy.c
>
>diff --git a/include/hw/vfio/vfio-container.h
>b/include/hw/vfio/vfio-container.h
>index afc498d..21e5807 100644
>--- a/include/hw/vfio/vfio-container.h
>+++ b/include/hw/vfio/vfio-container.h
>@@ -10,6 +10,7 @@
> #define HW_VFIO_CONTAINER_H
>
> #include "hw/vfio/vfio-container-base.h"
>+#include "hw/vfio/vfio-cpr.h"
Now that we have this change, may we remove #include of vfio-cpr.h in
hw/vfio/container.c?
Maybe this belong to patch8?
>
> typedef struct VFIOContainer VFIOContainer;
> typedef struct VFIODevice VFIODevice;
>@@ -29,6 +30,7 @@ typedef struct VFIOContainer {
> int fd; /* /dev/vfio/vfio, empowered by the attached groups */
> unsigned iommu_type;
> QLIST_HEAD(, VFIOGroup) group_list;
>+VFIOContainerCPR cpr;
> } VFIOContainer;
>
> OBJECT_DECLARE_SIMPLE_TYPE(VFIOContainer, VFIO_IOMMU_LEGACY);
>diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
>index 750ea5b..d4e0bd5 100644
>--- a/include/hw/vfio/vfio-cpr.h
>+++ b/include/hw/vfio/vfio-cpr.h
>@@ -9,8 +9,23 @@
> #ifndef HW_VFIO_VFIO_CPR_H
> #define HW_VFIO_VFIO_CPR_H
>
>+#include "migration/misc.h"
>+
>+struct VFIOContainer;
> struct VFIOContainerBase;
>
>+typedef struct VFIOContainerCPR {
>+Error *blocker;
>+} VFIOContainerCPR;
>+
>+
>+bool vfio_legacy_cpr_register_container(struct VFIOContainer *container,
>+Error **errp);
>+void vfio_legacy_cpr_unregister_container(struct VFIOContainer *container);
>+
>+int vfio_cpr_reboot_notifier(NotifierWithReturn *notifier, MigrationEvent *e,
>+ Error **errp);
>+
> bool vfio_cpr_register_container(struct VFIOContainerBase *bcontainer,
> Error **errp);
> void vfio_cpr_unregister_container(struct VFIOContainerBase *bcontainer);
>diff --git a/hw/vfio/container.c b/hw/vfio/container.c
>index 0f948d0..7d2035c 100644
>--- a/hw/vfio/container.c
>+++ b/hw/vfio/container.c
>@@ -643,7 +643,7 @@ static bool vfio_container_connect(VFIOGroup *group,
>AddressSpace *as,
> new_container = true;
> bcontainer = &container->bcontainer;
>
>-if (!vfio_cpr_register_container(bcontainer, errp)) {
>+if (!vfio_legacy_cpr_register_container(container, errp)) {
> goto fail;
> }
>
>@@ -679,7 +679,7 @@ fail:
> vioc->release(bcontainer);
> }
> if (new_container) {
>-vfio_cpr_unregister_container(bcontainer);
>+vfio_legacy_cpr_unregister_container(container);
> object_unref(container);
> }
> if (fd >= 0) {
>@@ -720,7 +720,7 @@ static void vfio_container_disconnect(VFIOGroup *group)
> VFIOAddressSpace *space = bcontainer->space;
>
> trace_vfio_container_disconnect(container->fd);
>-vfio_cpr_unregister_container(bcontainer);
>+vfio_legacy_cpr_unregister_container(container);
> close(container->fd);
> object_unref(container);
>
>diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c
>new file mode 100644
>index 000..419b9fb
>--- /dev/null
>+++ b/hw/vfio/cpr-legacy.c
>@@ -0,0 +1,69 @@
>+/*
>+ * Copyright (c) 2021-2025 Oracle and/or its affiliates.
>+ *
>+ * SPDX-License-Identifier: GPL-2.0-or-later
>+ */
>+
>+#include
>+#include
>+#include "qemu/osdep.h"
>+#include "hw/vfio/vfio-container.h"
>+#include "hw/vfio/vfio-cpr.h"
Ditto.
>+#include "migration/blocker.h"
>+#include "migration/cpr.h"
>+#include "migration/migration.h"
>+#include "migration/vmstate.h"
>+#include "qapi/error.h"
>+
>+static bool vfio_cpr_supported(VFIOContainer *container, Error **errp)
>+{
>+if (!ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UPDATE_VADDR)) {
>+error_setg(errp, "VFIO container does not support VFIO_UPDATE_VADDR");
>+return false;
>+
>+} else if (!ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UNMAP_ALL)) {
>+error_setg(errp, "VFIO container does not support VFIO_UNMAP_ALL");
>+return false;
>+
>+} else {
>+return true;
>+}
>+}
>+
>+static const VMStateDescription vfio_container_vmstate = {
>+.name = "vfio-container",
>+.version_id = 0,
>+
Re: [PATCH V4 09/43] vfio/container: register container for cpr
On 5/29/25 21:24, Steve Sistare wrote:
Register a legacy container for cpr-transfer, replacing the generic CPR
register call with a more specific legacy container register call. Add a
blocker if the kernel does not support VFIO_UPDATE_VADDR or VFIO_UNMAP_ALL.
This is mostly boiler plate. The fields to to saved and restored are added
in subsequent patches.
Signed-off-by: Steve Sistare
Reviewed-by: Cédric Le Goater
Thanks,
C.
---
include/hw/vfio/vfio-container.h | 2 ++
include/hw/vfio/vfio-cpr.h | 15 +
hw/vfio/container.c | 6 ++--
hw/vfio/cpr-legacy.c | 69
hw/vfio/cpr.c| 5 ++-
hw/vfio/meson.build | 1 +
6 files changed, 92 insertions(+), 6 deletions(-)
create mode 100644 hw/vfio/cpr-legacy.c
diff --git a/include/hw/vfio/vfio-container.h b/include/hw/vfio/vfio-container.h
index afc498d..21e5807 100644
--- a/include/hw/vfio/vfio-container.h
+++ b/include/hw/vfio/vfio-container.h
@@ -10,6 +10,7 @@
#define HW_VFIO_CONTAINER_H
#include "hw/vfio/vfio-container-base.h"
+#include "hw/vfio/vfio-cpr.h"
typedef struct VFIOContainer VFIOContainer;
typedef struct VFIODevice VFIODevice;
@@ -29,6 +30,7 @@ typedef struct VFIOContainer {
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
unsigned iommu_type;
QLIST_HEAD(, VFIOGroup) group_list;
+VFIOContainerCPR cpr;
} VFIOContainer;
OBJECT_DECLARE_SIMPLE_TYPE(VFIOContainer, VFIO_IOMMU_LEGACY);
diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
index 750ea5b..d4e0bd5 100644
--- a/include/hw/vfio/vfio-cpr.h
+++ b/include/hw/vfio/vfio-cpr.h
@@ -9,8 +9,23 @@
#ifndef HW_VFIO_VFIO_CPR_H
#define HW_VFIO_VFIO_CPR_H
+#include "migration/misc.h"
+
+struct VFIOContainer;
struct VFIOContainerBase;
+typedef struct VFIOContainerCPR {
+Error *blocker;
+} VFIOContainerCPR;
+
+
+bool vfio_legacy_cpr_register_container(struct VFIOContainer *container,
+Error **errp);
+void vfio_legacy_cpr_unregister_container(struct VFIOContainer *container);
+
+int vfio_cpr_reboot_notifier(NotifierWithReturn *notifier, MigrationEvent *e,
+ Error **errp);
+
bool vfio_cpr_register_container(struct VFIOContainerBase *bcontainer,
Error **errp);
void vfio_cpr_unregister_container(struct VFIOContainerBase *bcontainer);
diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 0f948d0..7d2035c 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -643,7 +643,7 @@ static bool vfio_container_connect(VFIOGroup *group,
AddressSpace *as,
new_container = true;
bcontainer = &container->bcontainer;
-if (!vfio_cpr_register_container(bcontainer, errp)) {
+if (!vfio_legacy_cpr_register_container(container, errp)) {
goto fail;
}
@@ -679,7 +679,7 @@ fail:
vioc->release(bcontainer);
}
if (new_container) {
-vfio_cpr_unregister_container(bcontainer);
+vfio_legacy_cpr_unregister_container(container);
object_unref(container);
}
if (fd >= 0) {
@@ -720,7 +720,7 @@ static void vfio_container_disconnect(VFIOGroup *group)
VFIOAddressSpace *space = bcontainer->space;
trace_vfio_container_disconnect(container->fd);
-vfio_cpr_unregister_container(bcontainer);
+vfio_legacy_cpr_unregister_container(container);
close(container->fd);
object_unref(container);
diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c
new file mode 100644
index 000..419b9fb
--- /dev/null
+++ b/hw/vfio/cpr-legacy.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2021-2025 Oracle and/or its affiliates.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include
+#include
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-container.h"
+#include "hw/vfio/vfio-cpr.h"
+#include "migration/blocker.h"
+#include "migration/cpr.h"
+#include "migration/migration.h"
+#include "migration/vmstate.h"
+#include "qapi/error.h"
+
+static bool vfio_cpr_supported(VFIOContainer *container, Error **errp)
+{
+if (!ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UPDATE_VADDR)) {
+error_setg(errp, "VFIO container does not support VFIO_UPDATE_VADDR");
+return false;
+
+} else if (!ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UNMAP_ALL)) {
+error_setg(errp, "VFIO container does not support VFIO_UNMAP_ALL");
+return false;
+
+} else {
+return true;
+}
+}
+
+static const VMStateDescription vfio_container_vmstate = {
+.name = "vfio-container",
+.version_id = 0,
+.minimum_version_id = 0,
+.needed = cpr_incoming_needed,
+.fields = (VMStateField[]) {
+VMSTATE_END_OF_LIST()
+}
+};
+
+bool vfio_legacy_cpr_register_container(VFIOContainer *container, Error **errp)
+{
+VFIOContainerBase
