rdma_registration_stop() runs after every ram_save_iterate() call, which can stop before a complete RAM scan because of rate limiting. Draining there ties the completion barrier to a SaveVM section rather than the point where the scan wraps and a newer page version can be sent.
Register an RDMA RAM round notifier that drains all outstanding writes. Keep posting the buffered tail in rdma_registration_stop() so it remains ordered before REGISTER_FINISHED. Signed-off-by: Yanfei Xu <[email protected]> Reviewed-by: Peter Xu <[email protected]> --- migration/rdma.c | 58 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/migration/rdma.c b/migration/rdma.c index b8a0e39d2a..16e9c6c30e 100644 --- a/migration/rdma.c +++ b/migration/rdma.c @@ -2627,21 +2627,16 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc, return done; } -/* - * Block until all the outstanding chunks have been delivered by the hardware. - */ -static int qemu_rdma_drain_cq(RDMAContext *rdma) +/* Block until all outstanding writes have been delivered by the hardware. */ +static int qemu_rdma_drain_cq(RDMAContext *rdma, Error **errp) { - Error *err = NULL; - - if (qemu_rdma_write_flush(rdma, &err) < 0) { - error_report_err(err); + if (qemu_rdma_write_flush(rdma, errp) < 0) { return -1; } while (rdma->nb_sent) { if (qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL) < 0) { - error_report("rdma migration: complete polling error!"); + error_setg(errp, "rdma migration: completion polling error"); return -1; } } @@ -2649,6 +2644,38 @@ static int qemu_rdma_drain_cq(RDMAContext *rdma) return 0; } +static int rdma_ram_round_notify(NotifierWithReturn *n G_GNUC_UNUSED, + void *data, Error **errp) +{ + RAMRoundNotifyData *round_data = data; + QEMUFile *f; + QIOChannelRDMA *rioc; + RDMAContext *rdma; + + if (!migrate_rdma()) { + return 0; + } + + f = round_data->file; + RCU_READ_LOCK_GUARD(); + rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f)); + rdma = qatomic_rcu_read(&rioc->rdmaout); + if (!rdma) { + error_setg(errp, "RDMA output context is not set"); + return -1; + } + + if (rdma_errored(rdma)) { + error_setg(errp, "RDMA is in an error state"); + return -1; + } + + return qemu_rdma_drain_cq(rdma, errp); +} + +static NotifierWithReturn rdma_ram_round_notifier = { + .notify = rdma_ram_round_notify, +}; static int qio_channel_rdma_set_blocking(QIOChannel *ioc, bool blocking, @@ -3549,7 +3576,8 @@ int rdma_registration_start(QEMUFile *f, uint64_t flags) /* * Inform dest that dynamic registrations are done for now. - * First, flush writes, if any. + * Post any buffered write before sending the control marker. Outstanding + * writes are drained at the RAM scan round boundary. */ int rdma_registration_stop(QEMUFile *f, uint64_t flags) { @@ -3575,9 +3603,10 @@ int rdma_registration_stop(QEMUFile *f, uint64_t flags) } qemu_fflush(f); - ret = qemu_rdma_drain_cq(rdma); + ret = qemu_rdma_write_flush(rdma, &err); if (ret < 0) { + error_report_err(err); goto err; } @@ -3711,6 +3740,13 @@ static void qio_channel_rdma_register_types(void) type_init(qio_channel_rdma_register_types); +static void rdma_register_migration_notifiers(void) +{ + ram_round_add_notifier(&rdma_ram_round_notifier); +} + +migration_init(rdma_register_migration_notifiers); + static QIOChannel *rdma_new_input(RDMAContext *rdma) { QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(object_new(TYPE_QIO_CHANNEL_RDMA)); -- 2.20.1
