[PATCH] hw/virtio: Add support for VDPA network simulation devices
This patch adds support for VDPA network simulation devices. The device is developed based on virtio-net and tap backend, and supports hardware live migration function. For more details, please refer to "docs/system/devices/vdpa-net.rst" Signed-off-by: Hao Chen --- MAINTAINERS | 5 + docs/system/device-emulation.rst| 1 + docs/system/devices/vdpa-net.rst| 121 + hw/net/virtio-net.c | 16 ++ hw/virtio/virtio-pci.c | 189 +++- hw/virtio/virtio.c | 39 include/hw/virtio/virtio-pci.h | 5 + include/hw/virtio/virtio.h | 19 ++ include/standard-headers/linux/virtio_pci.h | 7 + 9 files changed, 399 insertions(+), 3 deletions(-) create mode 100644 docs/system/devices/vdpa-net.rst diff --git a/MAINTAINERS b/MAINTAINERS index 7d61fb9319..a1bde36bb0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2359,6 +2359,11 @@ F: hw/virtio/vhost-user-scmi* F: include/hw/virtio/vhost-user-scmi.h F: tests/qtest/libqos/virtio-scmi.* +vdpa-net +M: Hao Chen +S: Maintained +F: docs/system/devices/vdpa-net.rst + virtio-crypto M: Gonglei S: Supported diff --git a/docs/system/device-emulation.rst b/docs/system/device-emulation.rst index f19777411c..e4a27f53c8 100644 --- a/docs/system/device-emulation.rst +++ b/docs/system/device-emulation.rst @@ -99,3 +99,4 @@ Emulated Devices devices/canokey.rst devices/usb-u2f.rst devices/igb.rst + devices/vdpa-net.rst diff --git a/docs/system/devices/vdpa-net.rst b/docs/system/devices/vdpa-net.rst new file mode 100644 index 00..323d8c926a --- /dev/null +++ b/docs/system/devices/vdpa-net.rst @@ -0,0 +1,121 @@ +vdpa net + + +This document explains the setup and usage of the vdpa network device. +The vdpa network device is a paravirtualized vdpa emulate device. + +Description +--- + +VDPA net devices support dirty page bitmap mark and vring state saving and recovery. + +Users can use this VDPA device for live migration simulation testing in a nested virtualization environment. + +Registers layout + + +The vdpa device add live migrate registers layout as follow:: + + Offset Register Name Bitwidth Associated vq + 0x0 LM_LOGGING_CTRL 4bits + 0x10 LM_BASE_ADDR_LOW 32bits + 0x14 LM_BASE_ADDR_HIGH32bits + 0x18 LM_END_ADDR_LOW 32bits + 0x1c LM_END_ADDR_HIGH 32bits + 0x20 LM_RING_STATE_OFFSET32bits vq0 + 0x24 LM_RING_STATE_OFFSET32bits vq1 + 0x28 LM_RING_STATE_OFFSET32bits vq2 + .. + 0x20+1023*4 LM_RING_STATE_OFFSET 32bits vq1023 + +These registers are extended at the end of the notify bar space. + +Architecture diagram + +:: + + || + | guest-L1-user-space| + || + | || + | | [virtio-net driver] | + | | ^ guest-L2-src(iommu=on) | + | |--|-| + | | | qemu-L2-src(viommu)| + | [dpdk-vdpa]<->[vhost socket]<-+->[vhost-user backend(iommu=on)]| + -- + -- + | ^ guest-L1-kernel-space | + | || + |[VFIO] | + | ^| + | | guest-L1-src(iommu=on) | + |- + |- + | [vdpa net device(iommu=on)][manager nic device]| + | ||| + | ||| + | [tap device] qemu-L1-src(viommu) || + +- + | + | +- | +| kernel net bridge |<- +| virbr0 |<-
Re: [PATCH] hw/virtio: Add support for VDPA network simulation devices
On 2024-02-21 15:38:02+0800, Hao Chen wrote: > This patch adds support for VDPA network simulation devices. > The device is developed based on virtio-net and tap backend, > and supports hardware live migration function. > > For more details, please refer to "docs/system/devices/vdpa-net.rst" > > Signed-off-by: Hao Chen > --- > MAINTAINERS | 5 + > docs/system/device-emulation.rst| 1 + > docs/system/devices/vdpa-net.rst| 121 + > hw/net/virtio-net.c | 16 ++ > hw/virtio/virtio-pci.c | 189 +++- > hw/virtio/virtio.c | 39 > include/hw/virtio/virtio-pci.h | 5 + > include/hw/virtio/virtio.h | 19 ++ > include/standard-headers/linux/virtio_pci.h | 7 + > 9 files changed, 399 insertions(+), 3 deletions(-) > create mode 100644 docs/system/devices/vdpa-net.rst [..] > diff --git a/include/standard-headers/linux/virtio_pci.h > b/include/standard-headers/linux/virtio_pci.h > index b7fdfd0668..fb5391cef6 100644 > --- a/include/standard-headers/linux/virtio_pci.h > +++ b/include/standard-headers/linux/virtio_pci.h > @@ -216,6 +216,13 @@ struct virtio_pci_cfg_cap { > #define VIRTIO_PCI_COMMON_Q_NDATA56 > #define VIRTIO_PCI_COMMON_Q_RESET58 > > +#define LM_LOGGING_CTRL 0 > +#define LM_BASE_ADDR_LOW4 > +#define LM_BASE_ADDR_HIGH 8 > +#define LM_END_ADDR_LOW 12 > +#define LM_END_ADDR_HIGH16 > +#define LM_VRING_STATE_OFFSET 0x20 These changes are not in upstream Linux and will be undone by ./scripts/update-linux-headers.sh. Are they intentionally in this header? > + > #endif /* VIRTIO_PCI_NO_MODERN */ > > #endif
Re: [PATCH] hw/virtio: Add support for VDPA network simulation devices
On Wed, Mar 13, 2024 at 07:51:08PM +0100, Thomas Weißschuh wrote: > On 2024-02-21 15:38:02+0800, Hao Chen wrote: > > This patch adds support for VDPA network simulation devices. > > The device is developed based on virtio-net and tap backend, > > and supports hardware live migration function. > > > > For more details, please refer to "docs/system/devices/vdpa-net.rst" > > > > Signed-off-by: Hao Chen > > --- > > MAINTAINERS | 5 + > > docs/system/device-emulation.rst| 1 + > > docs/system/devices/vdpa-net.rst| 121 + > > hw/net/virtio-net.c | 16 ++ > > hw/virtio/virtio-pci.c | 189 +++- > > hw/virtio/virtio.c | 39 > > include/hw/virtio/virtio-pci.h | 5 + > > include/hw/virtio/virtio.h | 19 ++ > > include/standard-headers/linux/virtio_pci.h | 7 + > > 9 files changed, 399 insertions(+), 3 deletions(-) > > create mode 100644 docs/system/devices/vdpa-net.rst > > [..] > > > diff --git a/include/standard-headers/linux/virtio_pci.h > > b/include/standard-headers/linux/virtio_pci.h > > index b7fdfd0668..fb5391cef6 100644 > > --- a/include/standard-headers/linux/virtio_pci.h > > +++ b/include/standard-headers/linux/virtio_pci.h > > @@ -216,6 +216,13 @@ struct virtio_pci_cfg_cap { > > #define VIRTIO_PCI_COMMON_Q_NDATA 56 > > #define VIRTIO_PCI_COMMON_Q_RESET 58 > > > > +#define LM_LOGGING_CTRL 0 > > +#define LM_BASE_ADDR_LOW4 > > +#define LM_BASE_ADDR_HIGH 8 > > +#define LM_END_ADDR_LOW 12 > > +#define LM_END_ADDR_HIGH16 > > +#define LM_VRING_STATE_OFFSET 0x20 > > These changes are not in upstream Linux and will be undone by > ./scripts/update-linux-headers.sh. > > Are they intentionally in this header? Good point. Pls move. > > + > > #endif /* VIRTIO_PCI_NO_MODERN */ > > > > #endif
Re: [PATCH] hw/virtio: Add support for VDPA network simulation devices
On Thu, Mar 14, 2024 at 3:52 AM Michael S. Tsirkin wrote: > > On Wed, Mar 13, 2024 at 07:51:08PM +0100, Thomas Weißschuh wrote: > > On 2024-02-21 15:38:02+0800, Hao Chen wrote: > > > This patch adds support for VDPA network simulation devices. > > > The device is developed based on virtio-net and tap backend, > > > and supports hardware live migration function. > > > > > > For more details, please refer to "docs/system/devices/vdpa-net.rst" > > > > > > Signed-off-by: Hao Chen > > > --- > > > MAINTAINERS | 5 + > > > docs/system/device-emulation.rst| 1 + > > > docs/system/devices/vdpa-net.rst| 121 + > > > hw/net/virtio-net.c | 16 ++ > > > hw/virtio/virtio-pci.c | 189 +++- I think those modifications should belong to a separate file as it might conflict with virito features in the future. > > > hw/virtio/virtio.c | 39 > > > include/hw/virtio/virtio-pci.h | 5 + > > > include/hw/virtio/virtio.h | 19 ++ > > > include/standard-headers/linux/virtio_pci.h | 7 + > > > 9 files changed, 399 insertions(+), 3 deletions(-) > > > create mode 100644 docs/system/devices/vdpa-net.rst > > > > [..] > > > > > diff --git a/include/standard-headers/linux/virtio_pci.h > > > b/include/standard-headers/linux/virtio_pci.h > > > index b7fdfd0668..fb5391cef6 100644 > > > --- a/include/standard-headers/linux/virtio_pci.h > > > +++ b/include/standard-headers/linux/virtio_pci.h > > > @@ -216,6 +216,13 @@ struct virtio_pci_cfg_cap { > > > #define VIRTIO_PCI_COMMON_Q_NDATA 56 > > > #define VIRTIO_PCI_COMMON_Q_RESET 58 > > > > > > +#define LM_LOGGING_CTRL 0 > > > +#define LM_BASE_ADDR_LOW4 > > > +#define LM_BASE_ADDR_HIGH 8 > > > +#define LM_END_ADDR_LOW 12 > > > +#define LM_END_ADDR_HIGH16 > > > +#define LM_VRING_STATE_OFFSET 0x20 > > > > These changes are not in upstream Linux and will be undone by > > ./scripts/update-linux-headers.sh. > > > > Are they intentionally in this header? > > > Good point. Pls move. Right and this part, it's not a part of standard virtio. Thanks > > > > + > > > #endif /* VIRTIO_PCI_NO_MODERN */ > > > > > > #endif >
Re: [PATCH] hw/virtio: Add support for VDPA network simulation devices
On Thu, Mar 14, 2024 at 11:24:33AM +0800, Jason Wang wrote: > On Thu, Mar 14, 2024 at 3:52 AM Michael S. Tsirkin wrote: > > > > On Wed, Mar 13, 2024 at 07:51:08PM +0100, Thomas Weißschuh wrote: > > > On 2024-02-21 15:38:02+0800, Hao Chen wrote: > > > > This patch adds support for VDPA network simulation devices. > > > > The device is developed based on virtio-net and tap backend, > > > > and supports hardware live migration function. > > > > > > > > For more details, please refer to "docs/system/devices/vdpa-net.rst" > > > > > > > > Signed-off-by: Hao Chen > > > > --- > > > > MAINTAINERS | 5 + > > > > docs/system/device-emulation.rst| 1 + > > > > docs/system/devices/vdpa-net.rst| 121 + > > > > hw/net/virtio-net.c | 16 ++ > > > > hw/virtio/virtio-pci.c | 189 +++- > > I think those modifications should belong to a separate file as it > might conflict with virito features in the future. > > > > > hw/virtio/virtio.c | 39 > > > > include/hw/virtio/virtio-pci.h | 5 + > > > > include/hw/virtio/virtio.h | 19 ++ > > > > include/standard-headers/linux/virtio_pci.h | 7 + > > > > 9 files changed, 399 insertions(+), 3 deletions(-) > > > > create mode 100644 docs/system/devices/vdpa-net.rst > > > > > > [..] > > > > > > > diff --git a/include/standard-headers/linux/virtio_pci.h > > > > b/include/standard-headers/linux/virtio_pci.h > > > > index b7fdfd0668..fb5391cef6 100644 > > > > --- a/include/standard-headers/linux/virtio_pci.h > > > > +++ b/include/standard-headers/linux/virtio_pci.h > > > > @@ -216,6 +216,13 @@ struct virtio_pci_cfg_cap { > > > > #define VIRTIO_PCI_COMMON_Q_NDATA 56 > > > > #define VIRTIO_PCI_COMMON_Q_RESET 58 > > > > > > > > +#define LM_LOGGING_CTRL 0 > > > > +#define LM_BASE_ADDR_LOW4 > > > > +#define LM_BASE_ADDR_HIGH 8 > > > > +#define LM_END_ADDR_LOW 12 > > > > +#define LM_END_ADDR_HIGH16 > > > > +#define LM_VRING_STATE_OFFSET 0x20 > > > > > > These changes are not in upstream Linux and will be undone by > > > ./scripts/update-linux-headers.sh. > > > > > > Are they intentionally in this header? > > > > > > Good point. Pls move. > > Right and this part, it's not a part of standard virtio. > > Thanks I'm thinking of reverting this patch unless there's a resolution soon, and reapplying later after the release. > > > > > > + > > > > #endif /* VIRTIO_PCI_NO_MODERN */ > > > > > > > > #endif > >
Re: [PATCH] hw/virtio: Add support for VDPA network simulation devices
On Wed, Feb 21, 2024 at 03:38:02PM +0800, Hao Chen wrote: > This patch adds support for VDPA network simulation devices. > The device is developed based on virtio-net and tap backend, > and supports hardware live migration function. > > For more details, please refer to "docs/system/devices/vdpa-net.rst" > > Signed-off-by: Hao Chen I am not really inclined to merge this, virtio TC is now working on LM support that physical device can support, feel free to join that effort. Thanks! > --- > MAINTAINERS | 5 + > docs/system/device-emulation.rst| 1 + > docs/system/devices/vdpa-net.rst| 121 + > hw/net/virtio-net.c | 16 ++ > hw/virtio/virtio-pci.c | 189 +++- > hw/virtio/virtio.c | 39 > include/hw/virtio/virtio-pci.h | 5 + > include/hw/virtio/virtio.h | 19 ++ > include/standard-headers/linux/virtio_pci.h | 7 + > 9 files changed, 399 insertions(+), 3 deletions(-) > create mode 100644 docs/system/devices/vdpa-net.rst > > diff --git a/MAINTAINERS b/MAINTAINERS > index 7d61fb9319..a1bde36bb0 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -2359,6 +2359,11 @@ F: hw/virtio/vhost-user-scmi* > F: include/hw/virtio/vhost-user-scmi.h > F: tests/qtest/libqos/virtio-scmi.* > > +vdpa-net > +M: Hao Chen > +S: Maintained > +F: docs/system/devices/vdpa-net.rst > + > virtio-crypto > M: Gonglei > S: Supported > diff --git a/docs/system/device-emulation.rst > b/docs/system/device-emulation.rst > index f19777411c..e4a27f53c8 100644 > --- a/docs/system/device-emulation.rst > +++ b/docs/system/device-emulation.rst > @@ -99,3 +99,4 @@ Emulated Devices > devices/canokey.rst > devices/usb-u2f.rst > devices/igb.rst > + devices/vdpa-net.rst > diff --git a/docs/system/devices/vdpa-net.rst > b/docs/system/devices/vdpa-net.rst > new file mode 100644 > index 00..323d8c926a > --- /dev/null > +++ b/docs/system/devices/vdpa-net.rst > @@ -0,0 +1,121 @@ > +vdpa net > + > + > +This document explains the setup and usage of the vdpa network device. > +The vdpa network device is a paravirtualized vdpa emulate device. > + > +Description > +--- > + > +VDPA net devices support dirty page bitmap mark and vring state saving and > recovery. > + > +Users can use this VDPA device for live migration simulation testing in a > nested virtualization environment. > + > +Registers layout > + > + > +The vdpa device add live migrate registers layout as follow:: > + > + Offset Register Name Bitwidth Associated vq > + 0x0 LM_LOGGING_CTRL 4bits > + 0x10 LM_BASE_ADDR_LOW 32bits > + 0x14 LM_BASE_ADDR_HIGH32bits > + 0x18 LM_END_ADDR_LOW 32bits > + 0x1c LM_END_ADDR_HIGH 32bits > + 0x20 LM_RING_STATE_OFFSET 32bits vq0 > + 0x24 LM_RING_STATE_OFFSET 32bits vq1 > + 0x28 LM_RING_STATE_OFFSET 32bits vq2 > + .. > + 0x20+1023*4 LM_RING_STATE_OFFSET 32bits vq1023 > + > +These registers are extended at the end of the notify bar space. > + > +Architecture diagram > + > +:: > + > + || > + | guest-L1-user-space| > + || > + | || > + | | [virtio-net driver] | > + | | ^ guest-L2-src(iommu=on) | > + | |--|-| > + | | | qemu-L2-src(viommu)| > + | [dpdk-vdpa]<->[vhost socket]<-+->[vhost-user backend(iommu=on)]| > + -- > + -- > + | ^ guest-L1-kernel-space | > + | || > + |[VFIO] | > + | ^| > + | | guest-L1-src(iommu=on) | > + |- > + |- > + | [vdpa net device(iommu=on)][manager nic device]| > + | ||| > + | ||
Re: [PATCH] hw/virtio: Add support for VDPA network simulation devices
On Mon, Mar 18, 2024 at 8:41 PM Michael S. Tsirkin wrote: > > On Thu, Mar 14, 2024 at 11:24:33AM +0800, Jason Wang wrote: > > On Thu, Mar 14, 2024 at 3:52 AM Michael S. Tsirkin wrote: > > > > > > On Wed, Mar 13, 2024 at 07:51:08PM +0100, Thomas Weißschuh wrote: > > > > On 2024-02-21 15:38:02+0800, Hao Chen wrote: > > > > > This patch adds support for VDPA network simulation devices. > > > > > The device is developed based on virtio-net and tap backend, > > > > > and supports hardware live migration function. > > > > > > > > > > For more details, please refer to "docs/system/devices/vdpa-net.rst" > > > > > > > > > > Signed-off-by: Hao Chen > > > > > --- > > > > > MAINTAINERS | 5 + > > > > > docs/system/device-emulation.rst| 1 + > > > > > docs/system/devices/vdpa-net.rst| 121 + > > > > > hw/net/virtio-net.c | 16 ++ > > > > > hw/virtio/virtio-pci.c | 189 > > > > > +++- > > > > I think those modifications should belong to a separate file as it > > might conflict with virito features in the future. > > > > > > > hw/virtio/virtio.c | 39 > > > > > include/hw/virtio/virtio-pci.h | 5 + > > > > > include/hw/virtio/virtio.h | 19 ++ > > > > > include/standard-headers/linux/virtio_pci.h | 7 + > > > > > 9 files changed, 399 insertions(+), 3 deletions(-) > > > > > create mode 100644 docs/system/devices/vdpa-net.rst > > > > > > > > [..] > > > > > > > > > diff --git a/include/standard-headers/linux/virtio_pci.h > > > > > b/include/standard-headers/linux/virtio_pci.h > > > > > index b7fdfd0668..fb5391cef6 100644 > > > > > --- a/include/standard-headers/linux/virtio_pci.h > > > > > +++ b/include/standard-headers/linux/virtio_pci.h > > > > > @@ -216,6 +216,13 @@ struct virtio_pci_cfg_cap { > > > > > #define VIRTIO_PCI_COMMON_Q_NDATA 56 > > > > > #define VIRTIO_PCI_COMMON_Q_RESET 58 > > > > > > > > > > +#define LM_LOGGING_CTRL 0 > > > > > +#define LM_BASE_ADDR_LOW4 > > > > > +#define LM_BASE_ADDR_HIGH 8 > > > > > +#define LM_END_ADDR_LOW 12 > > > > > +#define LM_END_ADDR_HIGH16 > > > > > +#define LM_VRING_STATE_OFFSET 0x20 > > > > > > > > These changes are not in upstream Linux and will be undone by > > > > ./scripts/update-linux-headers.sh. > > > > > > > > Are they intentionally in this header? > > > > > > > > > Good point. Pls move. > > > > Right and this part, it's not a part of standard virtio. > > > > Thanks > > I'm thinking of reverting this patch unless there's a resolution > soon, and reapplying later after the release. I think we need to revert this and re-visit in the next release. Thanks > > > > > > > > > > + > > > > > #endif /* VIRTIO_PCI_NO_MODERN */ > > > > > > > > > > #endif > > > >