Track whole-migration counters for the RDMA send queue: peak nb_sent (max outstanding writes), the number of posts, how many posts hit a full send queue, and how many times the queue drained to zero. Reset at RAM_CONTROL_SETUP and dumped at RAM_CONTROL_FINISH; migration-thread only, no locking.
Signed-off-by: Dmitrii Omelchenko <[email protected]> --- migration/rdma.c | 60 ++++++++++++++++++++++++++++++++++++++++++ migration/trace-events | 1 + 2 files changed, 61 insertions(+) diff --git a/migration/rdma.c b/migration/rdma.c index 444123cbca..50f5d7b839 100644 --- a/migration/rdma.c +++ b/migration/rdma.c @@ -302,6 +302,12 @@ typedef struct RDMAContext { /* number of outstanding writes */ int nb_sent; + /* nb_sent statistics */ + uint64_t nbsent_max; /* peak outstanding writes */ + uint64_t nbsent_posts; /* posts (nb_sent++) */ + uint64_t nbsent_full_hits; /* posts that hit a full SQ */ + uint64_t nbsent_zero_hits; /* drains to zero */ + /* store info about current buffer so that we can merge it with future sends */ uint64_t current_addr; @@ -1164,6 +1170,11 @@ static uint64_t qemu_rdma_make_wrid(uint64_t wr_id, uint64_t index, return result; } +/* nb_sent statistics counters */ +static void rdma_nbsent_inc(RDMAContext *rdma, uint64_t nb_sent); +static void rdma_nbsent_full(RDMAContext *rdma); +static void rdma_nbsent_zero(RDMAContext *rdma); + /* * Consult the connection manager to see a work request * (of any kind) has completed. @@ -1213,6 +1224,9 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct ibv_cq *cq, if (rdma->nb_sent > 0) { rdma->nb_sent--; + if (rdma->nb_sent == 0) { + rdma_nbsent_zero(rdma); + } } } else { trace_rdma_poll_other(wr_id, rdma->nb_sent); @@ -1438,6 +1452,7 @@ static bool qemu_rdma_post_send(RDMAContext *rdma, ret = ibv_post_send(rdma->qp, send_wr, &bad_wr); if (ret == ENOMEM && rdma->nb_sent) { trace_rdma_post_send_queue_full(send_wr->wr_id, rdma->nb_sent); + rdma_nbsent_full(rdma); ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); if (ret < 0) { error_setg(errp, "rdma migration: failed to make room for %s", @@ -1953,6 +1968,7 @@ static int qemu_rdma_write_flush(RDMAContext *rdma, Error **errp) if (ret == 0) { rdma->nb_sent++; trace_rdma_write_flush(rdma->nb_sent); + rdma_nbsent_inc(rdma, rdma->nb_sent); } rdma->current_length = 0; @@ -2998,6 +3014,41 @@ err: return -1; } +static void rdma_nbsent_inc(RDMAContext *rdma, uint64_t nb_sent) +{ + rdma->nbsent_posts++; + if (nb_sent > rdma->nbsent_max) { + rdma->nbsent_max = nb_sent; + } +} + +static void rdma_nbsent_full(RDMAContext *rdma) +{ + rdma->nbsent_full_hits++; +} + +static void rdma_nbsent_zero(RDMAContext *rdma) +{ + rdma->nbsent_zero_hits++; +} + +static void rdma_stats_reset(RDMAContext *rdma) +{ + rdma->nbsent_max = 0; + rdma->nbsent_posts = 0; + rdma->nbsent_full_hits = 0; + rdma->nbsent_zero_hits = 0; +} + +static void rdma_nbsent_stats_dump(RDMAContext *rdma) +{ + if (rdma->nbsent_posts == 0) { + return; + } + trace_rdma_nbsent_stats(rdma->nbsent_max, rdma->nbsent_posts, + rdma->nbsent_full_hits, rdma->nbsent_zero_hits); +} + int rdma_control_save_page(QEMUFile *f, ram_addr_t block_offset, ram_addr_t offset, size_t size) { @@ -3547,6 +3598,10 @@ int rdma_registration_start(QEMUFile *f, uint64_t flags) return -1; } + if (flags == RAM_CONTROL_SETUP) { + rdma_stats_reset(rdma); + } + trace_rdma_registration_start(flags); qemu_put_be64(f, RAM_SAVE_FLAG_HOOK); return qemu_fflush(f); @@ -3581,6 +3636,11 @@ int rdma_registration_stop(QEMUFile *f, uint64_t flags) } qemu_fflush(f); + + if (flags == RAM_CONTROL_FINISH) { + rdma_nbsent_stats_dump(rdma); + } + ret = qemu_rdma_write_flush(rdma, &err); if (ret < 0) { diff --git a/migration/trace-events b/migration/trace-events index db911fe6f2..f8ed6a7250 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -245,6 +245,7 @@ rdma_resolve_host_trying(const char *host, const char *ip) "Trying %s => %s" rdma_write_flush(int sent) "sent total: %d" rdma_write_one_post(uint64_t chunk, long addr, long remote, uint32_t len) "Posting chunk: %" PRIu64 ", addr: 0x%lx remote: 0x%lx, bytes %" PRIu32 rdma_post_send_queue_full(uint64_t wr_id, int sent) "send queue full, wr_id=%" PRIu64 ", outstanding writes=%d" +rdma_nbsent_stats(uint64_t max, uint64_t posts, uint64_t full_hits, uint64_t zero_hits) "nb_sent max %" PRIu64 " posts %" PRIu64 " full-queue hits %" PRIu64 " drains to zero %" PRIu64 rdma_write_one_recvregres(int mykey, int theirkey, uint64_t chunk) "Received registration result: my key: 0x%x their key 0x%x, chunk %" PRIu64 rdma_write_one_sendreg(uint64_t chunk, int len, int index, int64_t offset) "Sending registration request chunk %" PRIu64 " for %d bytes, index: %d, offset: %" PRId64 rdma_write_one_top(uint64_t chunks, uint64_t size) "Writing %" PRIu64 " chunks, (%" PRIu64 " MB)" -- 2.39.5
