qemu_rdma_write_one() waits for an earlier write to the same registration chunk to complete. This serializes disjoint dirty ranges in a chunk and leaves send queue capacity unused.
Remove the per-chunk wait and use the reference counts to track all outstanding writes. The existing per-iteration drain remains the completion barrier. Signed-off-by: Yanfei Xu <[email protected]> --- migration/rdma.c | 19 +------------------ migration/trace-events | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/migration/rdma.c b/migration/rdma.c index 973a7a745a..63bc357657 100644 --- a/migration/rdma.c +++ b/migration/rdma.c @@ -1869,7 +1869,7 @@ static int qemu_rdma_write_one(RDMAContext *rdma, struct ibv_sge sge; struct ibv_send_wr send_wr = { 0 }; struct ibv_send_wr *bad_wr; - int reg_result_idx, ret, count = 0; + int reg_result_idx, ret; uint64_t chunk, chunks; uint64_t chunk_size = migrate_rdma_chunk_size(); uint8_t *chunk_start, *chunk_end; @@ -1910,23 +1910,6 @@ retry: chunk_end = ram_chunk_end(block, chunk + chunks); - - while (qemu_rdma_chunk_in_transit(block, chunk)) { - (void)count; - trace_qemu_rdma_write_one_block(count++, current_index, chunk, - sge.addr, length, rdma->nb_sent, block->nb_chunks); - - ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); - - if (ret < 0) { - error_setg(errp, "Failed to Wait for previous write to complete " - "block %d chunk %" PRIu64 - " current %" PRIu64 " len %" PRIu64 " %d", - current_index, chunk, sge.addr, length, rdma->nb_sent); - return -1; - } - } - if (!rdma->pin_all || !block->is_ram_block) { if (!block->remote_keys[chunk]) { /* diff --git a/migration/trace-events b/migration/trace-events index 172761be78..253ff71891 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -249,7 +249,6 @@ qemu_rdma_unregister_waiting_proc(uint64_t chunk, int pos) "Processing unregiste qemu_rdma_unregister_waiting_send(uint64_t chunk) "Sending unregister for chunk: %" PRIu64 qemu_rdma_unregister_waiting_complete(uint64_t chunk) "Unregister for chunk: %" PRIu64 " complete." qemu_rdma_write_flush(int sent) "sent total: %d" -qemu_rdma_write_one_block(int count, int block, uint64_t chunk, uint64_t current, uint64_t len, int nb_sent, int nb_chunks) "(%d) Not clobbering: block: %d chunk %" PRIu64 " current %" PRIu64 " len %" PRIu64 " %d %d" qemu_rdma_write_one_post(uint64_t chunk, long addr, long remote, uint32_t len) "Posting chunk: %" PRIu64 ", addr: 0x%lx remote: 0x%lx, bytes %" PRIu32 qemu_rdma_write_one_queue_full(void) "" qemu_rdma_write_one_recvregres(int mykey, int theirkey, uint64_t chunk) "Received registration result: my key: 0x%x their key 0x%x, chunk %" PRIu64 -- 2.20.1
