On Sun, 21 Jun 2020 01:51:18 +0530 Kirti Wankhede <kwankh...@nvidia.com> wrote:
> Sequence during _RESUMING device state: > While data for this device is available, repeat below steps: > a. read data_offset from where user application should write data. > b. write data of data_size to migration region from data_offset. > c. write data_size which indicates vendor driver that data is written in > staging buffer. > > For user, data is opaque. User should write data in the same order as > received. > > Signed-off-by: Kirti Wankhede <kwankh...@nvidia.com> > Reviewed-by: Neo Jia <c...@nvidia.com> > Reviewed-by: Dr. David Alan Gilbert <dgilb...@redhat.com> > --- > hw/vfio/migration.c | 177 > +++++++++++++++++++++++++++++++++++++++++++++++++++ > hw/vfio/trace-events | 3 + > 2 files changed, 180 insertions(+) > > diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c > index ef1150c1ff02..faacea5327cb 100644 > --- a/hw/vfio/migration.c > +++ b/hw/vfio/migration.c > @@ -302,6 +302,33 @@ static int vfio_save_device_config_state(QEMUFile *f, > void *opaque) > return qemu_file_get_error(f); > } > > +static int vfio_load_device_config_state(QEMUFile *f, void *opaque) > +{ > + VFIODevice *vbasedev = opaque; > + uint64_t data; > + > + if (vbasedev->ops && vbasedev->ops->vfio_load_config) { > + int ret; > + > + ret = vbasedev->ops->vfio_load_config(vbasedev, f); > + if (ret) { > + error_report("%s: Failed to load device config space", > + vbasedev->name); > + return ret; > + } > + } > + > + data = qemu_get_be64(f); > + if (data != VFIO_MIG_FLAG_END_OF_STATE) { > + error_report("%s: Failed loading device config space, " > + "end flag incorrect 0x%"PRIx64, vbasedev->name, data); > + return -EINVAL; > + } > + > + trace_vfio_load_device_config_state(vbasedev->name); > + return qemu_file_get_error(f); > +} > + > /* ---------------------------------------------------------------------- */ > > static int vfio_save_setup(QEMUFile *f, void *opaque) > @@ -472,12 +499,162 @@ static int vfio_save_complete_precopy(QEMUFile *f, > void *opaque) > return ret; > } > > +static int vfio_load_setup(QEMUFile *f, void *opaque) > +{ > + VFIODevice *vbasedev = opaque; > + VFIOMigration *migration = vbasedev->migration; > + int ret = 0; > + > + if (migration->region.mmaps) { > + ret = vfio_region_mmap(&migration->region); > + if (ret) { > + error_report("%s: Failed to mmap VFIO migration region %d: %s", > + vbasedev->name, migration->region.nr, > + strerror(-ret)); > + return ret; Not fatal. > + } > + } > + > + ret = vfio_migration_set_state(vbasedev, ~VFIO_DEVICE_STATE_MASK, > + VFIO_DEVICE_STATE_RESUMING); > + if (ret) { > + error_report("%s: Failed to set state RESUMING", vbasedev->name); > + } > + return ret; > +} > + > +static int vfio_load_cleanup(void *opaque) > +{ > + vfio_save_cleanup(opaque); > + return 0; > +} > + > +static int vfio_load_state(QEMUFile *f, void *opaque, int version_id) > +{ > + VFIODevice *vbasedev = opaque; > + VFIOMigration *migration = vbasedev->migration; > + int ret = 0; > + uint64_t data, data_size; > + > + data = qemu_get_be64(f); > + while (data != VFIO_MIG_FLAG_END_OF_STATE) { > + > + trace_vfio_load_state(vbasedev->name, data); > + > + switch (data) { > + case VFIO_MIG_FLAG_DEV_CONFIG_STATE: > + { > + ret = vfio_load_device_config_state(f, opaque); > + if (ret) { > + return ret; > + } > + break; > + } > + case VFIO_MIG_FLAG_DEV_SETUP_STATE: > + { > + data = qemu_get_be64(f); > + if (data == VFIO_MIG_FLAG_END_OF_STATE) { > + return ret; > + } else { > + error_report("%s: SETUP STATE: EOS not found 0x%"PRIx64, > + vbasedev->name, data); > + return -EINVAL; This is essentially just a compatibility failure, right? For instance some future version of QEMU might include additional data between these markers that we don't understand and therefore we fail the migration. Thanks, Alex > + } > + break; > + } > + case VFIO_MIG_FLAG_DEV_DATA_STATE: > + { > + VFIORegion *region = &migration->region; > + uint64_t data_offset = 0, size; > + > + data_size = size = qemu_get_be64(f); > + if (data_size == 0) { > + break; > + } > + > + ret = pread(vbasedev->fd, &data_offset, sizeof(data_offset), > + region->fd_offset + > + offsetof(struct vfio_device_migration_info, > + data_offset)); > + if (ret != sizeof(data_offset)) { > + error_report("%s:Failed to get migration buffer data offset > %d", > + vbasedev->name, ret); > + return -EINVAL; > + } > + > + trace_vfio_load_state_device_data(vbasedev->name, data_offset, > + data_size); > + > + while (size) { > + void *buf = NULL; > + uint64_t sec_size; > + bool buffer_mmaped; > + > + buf = get_data_section_size(region, data_offset, size, > + &sec_size); > + > + buffer_mmaped = (buf != NULL); > + > + if (!buffer_mmaped) { > + buf = g_try_malloc(sec_size); > + if (!buf) { > + error_report("%s: Error allocating buffer ", > __func__); > + return -ENOMEM; > + } > + } > + > + qemu_get_buffer(f, buf, sec_size); > + > + if (!buffer_mmaped) { > + ret = pwrite(vbasedev->fd, buf, sec_size, > + region->fd_offset + data_offset); > + g_free(buf); > + > + if (ret != sec_size) { > + error_report("%s: Failed to set migration buffer %d", > + vbasedev->name, ret); > + return -EINVAL; > + } > + } > + size -= sec_size; > + data_offset += sec_size; > + } > + > + ret = pwrite(vbasedev->fd, &data_size, sizeof(data_size), > + region->fd_offset + > + offsetof(struct vfio_device_migration_info, > data_size)); > + if (ret != sizeof(data_size)) { > + error_report("%s: Failed to set migration buffer data size > %d", > + vbasedev->name, ret); > + return -EINVAL; > + } > + break; > + } > + > + default: > + error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data); > + return -EINVAL; > + } > + > + data = qemu_get_be64(f); > + ret = qemu_file_get_error(f); > + if (ret) { > + return ret; > + } > + } > + > + return ret; > +} > + > static SaveVMHandlers savevm_vfio_handlers = { > .save_setup = vfio_save_setup, > .save_cleanup = vfio_save_cleanup, > .save_live_pending = vfio_save_pending, > .save_live_iterate = vfio_save_iterate, > .save_live_complete_precopy = vfio_save_complete_precopy, > + .load_setup = vfio_load_setup, > + .load_cleanup = vfio_load_cleanup, > + .load_state = vfio_load_state, > }; > > /* ---------------------------------------------------------------------- */ > diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events > index 9a1c5e17d97f..4a4bd3ba9a2a 100644 > --- a/hw/vfio/trace-events > +++ b/hw/vfio/trace-events > @@ -157,3 +157,6 @@ vfio_save_device_config_state(const char *name) " (%s)" > vfio_save_pending(const char *name, uint64_t precopy, uint64_t postcopy, > uint64_t compatible) " (%s) precopy 0x%"PRIx64" postcopy 0x%"PRIx64" > compatible 0x%"PRIx64 > vfio_save_iterate(const char *name, int data_size) " (%s) data_size %d" > vfio_save_complete_precopy(const char *name) " (%s)" > +vfio_load_device_config_state(const char *name) " (%s)" > +vfio_load_state(const char *name, uint64_t data) " (%s) data 0x%"PRIx64 > +vfio_load_state_device_data(const char *name, uint64_t data_offset, uint64_t > data_size) " (%s) Offset 0x%"PRIx64" size 0x%"PRIx64