* Wolfgang Bumiller (w.bumil...@proxmox.com) wrote: > While testing with 4.0 we've run into issues with live migration from > 3.0.1 to 4.0 when a balloon device was involved. > > We'd see the following error on the destination: > qemu-system-x86_64: get_pci_config_device: Bad config data: i=0x10 read: a1 > device: 1 cmask: ff wmask: c0 w1cmask:0 > qemu-system-x86_64: Failed to load PCIDevice:config > qemu-system-x86_64: Failed to load virtio-balloon:virtio > qemu-system-x86_64: error while loading state for instance 0x0 of device > '0000:00:03.0/virtio-balloon' > qemu-system-x86_64: load of migration failed: Invalid argument > > After looking through the commits I noticed that the pci config sent for > the balloon device comes from > include/standard-headers/linux/virtio_balloon.h and changed size between > 3.1 and 4.0. > As a "guess" I tried reverting that change (commented out the two last > fields (and access to it in hw/virtio/virtio-balloon.c's > virtio_balloon_get_config()), and then the migration seems to go through > successfully. > > I've since also rebuilt qemu without our patches (tags v3.0.1 and v4.0.0) > and also tried with master (since dgilbert mentioned on irc remembering > the issue and that there may have been a fix around), but got the same > result. > > Posting here now as dgilbert requested on irc.
Apologies for the delay, I was out last week. Thanks for the command lines; I can confirm I can recreate it here using a simplified version of your command line. Dave > Here are the commands used to start qemu: > Source: > /usr/bin/kvm \ > -name randomclone \ > -chardev > 'socket,id=qmp,path=/var/run/qemu-server/101.qmp,server,nowait' \ > -mon 'chardev=qmp,mode=control' \ > -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ > -mon 'chardev=qmp-event,mode=control' \ > -pidfile /var/run/qemu-server/101.pid \ > -daemonize \ > -smbios 'type=1,uuid=f3ab31f6-ca7d-469c-bf51-547fd9bbd2d9' \ > -smp '4,sockets=1,cores=4,maxcpus=4' \ > -nodefaults \ > -boot > 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' > \ > -vnc unix:/var/run/qemu-server/101.vnc,password \ > -cpu host,+pcid,+spec-ctrl,+ssbd,+pdpe1gb,+kvm_pv_unhalt,+kvm_pv_eoi \ > -m 4096 \ > -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ > -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ > -device 'vmgenid,guid=fb282779-7056-4f1d-96bb-70f578294e45' \ > -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ > -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ > -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ > -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ > -iscsi 'initiator-name=iqn.1993-08.org.debian:01:856d32b504d' \ > -drive 'if=none,id=drive-ide2,media=cdrom,aio=threads' \ > -device > 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ > -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ > -drive > 'file=rbd:rbd/vm-101-disk-0:conf=/etc/pve/ceph.conf:id=admin:keyring=/etc/pve/priv/ceph/rbd.keyring,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=native,detect-zeroes=unmap' > \ > -device > 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,rotation_rate=1,bootindex=100' > \ > -netdev > 'type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' > \ > -device > 'virtio-net-pci,mac=4E:5D:50:75:4D:ED,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' > \ > -machine 'type=pc' \ > -enable-kvm > > Destination: > /usr/bin/kvm \ > -name randomclone \ > -chardev socket,id=qmp,path=/var/run/qemu-server/101.qmp,server,nowait \ > -mon chardev=qmp,mode=control \ > -chardev socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5 \ > -mon chardev=qmp-event,mode=control \ > -pidfile /var/run/qemu-server/101.pid \ > -smbios type=1,uuid=f3ab31f6-ca7d-469c-bf51-547fd9bbd2d9 \ > -smp 4,sockets=1,cores=4,maxcpus=4 \ > -nodefaults \ > -boot > menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg > \ > -vnc unix:/var/run/qemu-server/101.vnc,password \ > -cpu host,+pcid,+spec-ctrl,+ssbd,+pdpe1gb,+kvm_pv_unhalt,+kvm_pv_eoi \ > -m 4096 \ > -device pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e \ > -device pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f \ > -device vmgenid,guid=fb282779-7056-4f1d-96bb-70f578294e45 \ > -device piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2 \ > -device usb-tablet,id=tablet,bus=uhci.0,port=1 \ > -device VGA,id=vga,bus=pci.0,addr=0x2 \ > -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \ > -iscsi initiator-name=iqn.1993-08.org.debian:01:ee4e4a566b \ > -drive if=none,id=drive-ide2,media=cdrom,aio=threads \ > -device ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200 \ > -device virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5 \ > -drive > file=rbd:rbd/vm-101-disk-0:conf=/etc/pve/ceph.conf:id=admin:keyring=/etc/pve/priv/ceph/rbd.keyring,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=native,detect-zeroes=unmap > \ > -device > scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,rotation_rate=1,bootindex=100 > \ > -netdev > type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on > \ > -device > virtio-net-pci,mac=4E:5D:50:75:4D:ED,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300 > \ > -machine type=pc-i440fx-3.0 \ > -enable-kvm \ > -incoming tcp:10.9.2.106:9989 \ > -S > > This is the exact test-change I made which seems to work around it, but > a proper fix would be nicer. Not sure how, though. > > ---8<--- > diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c > index d96e4aa96f..8d631d67a8 100644 > --- a/hw/virtio/virtio-balloon.c > +++ b/hw/virtio/virtio-balloon.c > @@ -623,16 +623,16 @@ static void virtio_balloon_get_config(VirtIODevice > *vdev, uint8_t *config_data) > config.num_pages = cpu_to_le32(dev->num_pages); > config.actual = cpu_to_le32(dev->actual); > > - if (dev->free_page_report_status == FREE_PAGE_REPORT_S_REQUESTED) { > - config.free_page_report_cmd_id = > - cpu_to_le32(dev->free_page_report_cmd_id); > - } else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_STOP) { > - config.free_page_report_cmd_id = > - cpu_to_le32(VIRTIO_BALLOON_CMD_ID_STOP); > - } else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_DONE) { > - config.free_page_report_cmd_id = > - cpu_to_le32(VIRTIO_BALLOON_CMD_ID_DONE); > - } > + //if (dev->free_page_report_status == FREE_PAGE_REPORT_S_REQUESTED) { > + // config.free_page_report_cmd_id = > + // cpu_to_le32(dev->free_page_report_cmd_id); > + //} else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_STOP) { > + // config.free_page_report_cmd_id = > + // cpu_to_le32(VIRTIO_BALLOON_CMD_ID_STOP); > + //} else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_DONE) { > + // config.free_page_report_cmd_id = > + // cpu_to_le32(VIRTIO_BALLOON_CMD_ID_DONE); > + //} > > trace_virtio_balloon_get_config(config.num_pages, config.actual); > memcpy(config_data, &config, sizeof(struct virtio_balloon_config)); > diff --git a/include/standard-headers/linux/virtio_balloon.h > b/include/standard-headers/linux/virtio_balloon.h > index 9375ca2a70..86aca75972 100644 > --- a/include/standard-headers/linux/virtio_balloon.h > +++ b/include/standard-headers/linux/virtio_balloon.h > @@ -48,9 +48,9 @@ struct virtio_balloon_config { > /* Number of pages we've actually got in balloon. */ > uint32_t actual; > /* Free page report command id, readonly by guest */ > - uint32_t free_page_report_cmd_id; > - /* Stores PAGE_POISON if page poisoning is in use */ > - uint32_t poison_val; > + //uint32_t free_page_report_cmd_id; > + ///* Stores PAGE_POISON if page poisoning is in use */ > + //uint32_t poison_val; > }; > > #define VIRTIO_BALLOON_S_SWAP_IN 0 /* Amount of memory swapped in */ > -- Dr. David Alan Gilbert / dgilb...@redhat.com / Manchester, UK