Re: [Qemu-devel] [PATCH v6 07/11] hw/rdma: Free all receive buffers when QP is destroyed
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> When QP is destroyed the backend QP is destroyed as well. This ensures
> we clean all received buffer we posted to it.
> However, a contexts of these buffers are still remain in the device.
> Fix it by maintaining a list of buffer's context and free them when QP
> is destroyed.
>
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/rdma_backend.c | 26 --
> hw/rdma/rdma_backend.h | 2 +-
> hw/rdma/rdma_backend_defs.h | 2 +-
> hw/rdma/rdma_rm.c | 2 +-
> hw/rdma/rdma_utils.c| 29 +
> hw/rdma/rdma_utils.h| 11 +++
> 6 files changed, 63 insertions(+), 9 deletions(-)
>
> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
> index d0bbe57..e124d8d 100644
> --- a/hw/rdma/rdma_backend.c
> +++ b/hw/rdma/rdma_backend.c
> @@ -39,6 +39,7 @@
> typedef struct BackendCtx {
> void *up_ctx;
> struct ibv_sge sge; /* Used to save MAD recv buffer */
> +RdmaBackendQP *backend_qp; /* To maintain recv buffers */
> } BackendCtx;
>
> struct backend_umad {
> @@ -73,6 +74,7 @@ static void free_cqe_ctx(gpointer data, gpointer user_data)
> bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
> if (bctx) {
> rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
> +atomic_dec(&rdma_dev_res->stats.missing_cqe);
> }
> g_free(bctx);
> }
> @@ -85,13 +87,15 @@ static void clean_recv_mads(RdmaBackendDev *backend_dev)
> cqe_ctx_id = rdma_protected_qlist_pop_int64(&backend_dev->
> recv_mads_list);
> if (cqe_ctx_id != -ENOENT) {
> +atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
> free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
> backend_dev->rdma_dev_res);
> }
> } while (cqe_ctx_id != -ENOENT);
> }
>
> -static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq
> *ibcq)
> +static int rdma_poll_cq(RdmaBackendDev *backend_dev,
> +RdmaDeviceResources *rdma_dev_res, struct ibv_cq
> *ibcq)
> {
> int i, ne, total_ne = 0;
> BackendCtx *bctx;
> @@ -113,6 +117,8 @@ static int rdma_poll_cq(RdmaDeviceResources
> *rdma_dev_res, struct ibv_cq *ibcq)
>
> comp_handler(bctx->up_ctx, &wc[i]);
>
> +
> rdma_protected_gslist_remove_int32(&bctx->backend_qp->cqe_ctx_list,
> + wc[i].wr_id);
> rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
> g_free(bctx);
> }
> @@ -175,14 +181,12 @@ static void *comp_handler_thread(void *arg)
> }
>
> backend_dev->rdma_dev_res->stats.poll_cq_from_bk++;
> -rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq);
> +rdma_poll_cq(backend_dev, backend_dev->rdma_dev_res, ev_cq);
>
> ibv_ack_cq_events(ev_cq, 1);
> }
> }
>
> -/* TODO: Post cqe for all remaining buffs that were posted */
> -
> backend_dev->comp_thread.is_running = false;
>
> qemu_thread_exit(0);
> @@ -311,7 +315,7 @@ void rdma_backend_poll_cq(RdmaDeviceResources
> *rdma_dev_res, RdmaBackendCQ *cq)
> int polled;
>
> rdma_dev_res->stats.poll_cq_from_guest++;
> -polled = rdma_poll_cq(rdma_dev_res, cq->ibcq);
> +polled = rdma_poll_cq(cq->backend_dev, rdma_dev_res, cq->ibcq);
> if (!polled) {
> rdma_dev_res->stats.poll_cq_from_guest_empty++;
> }
> @@ -501,6 +505,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
>
> bctx = g_malloc0(sizeof(*bctx));
> bctx->up_ctx = ctx;
> +bctx->backend_qp = qp;
>
> rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
> if (unlikely(rc)) {
> @@ -508,6 +513,8 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
> goto err_free_bctx;
> }
>
> +rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
> +
> rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge,
> num_sge,
>&backend_dev->rdma_dev_res->stats.tx_len);
> if (rc) {
> @@ -616,6 +623,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
>
> bctx = g_malloc0(sizeof(*bctx));
> bctx->up_ctx = ctx;
> +bctx->backend_qp = qp;
>
> rc = rdma_rm_alloc_cqe_ctx(rdma_dev_res, &bctx_id, bctx);
> if (unlikely(rc)) {
> @@ -623,6 +631,8 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> goto err_free_bctx;
> }
>
> +rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
> +
> rc = build_host_sge_array(rdma_dev_res, new_sge, sge, num_sge,
>&backend_dev->rdma_dev_res->stats.rx_bufs_len);
> if (rc) {
> @@ -762,6 +772,8 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
> qp_type,
[Qemu-devel] [PATCH v6 07/11] hw/rdma: Free all receive buffers when QP is destroyed
When QP is destroyed the backend QP is destroyed as well. This ensures
we clean all received buffer we posted to it.
However, a contexts of these buffers are still remain in the device.
Fix it by maintaining a list of buffer's context and free them when QP
is destroyed.
Signed-off-by: Yuval Shaia
Reviewed-by: Marcel Apfelbaum
---
hw/rdma/rdma_backend.c | 26 --
hw/rdma/rdma_backend.h | 2 +-
hw/rdma/rdma_backend_defs.h | 2 +-
hw/rdma/rdma_rm.c | 2 +-
hw/rdma/rdma_utils.c| 29 +
hw/rdma/rdma_utils.h| 11 +++
6 files changed, 63 insertions(+), 9 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index d0bbe57..e124d8d 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -39,6 +39,7 @@
typedef struct BackendCtx {
void *up_ctx;
struct ibv_sge sge; /* Used to save MAD recv buffer */
+RdmaBackendQP *backend_qp; /* To maintain recv buffers */
} BackendCtx;
struct backend_umad {
@@ -73,6 +74,7 @@ static void free_cqe_ctx(gpointer data, gpointer user_data)
bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
if (bctx) {
rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
+atomic_dec(&rdma_dev_res->stats.missing_cqe);
}
g_free(bctx);
}
@@ -85,13 +87,15 @@ static void clean_recv_mads(RdmaBackendDev *backend_dev)
cqe_ctx_id = rdma_protected_qlist_pop_int64(&backend_dev->
recv_mads_list);
if (cqe_ctx_id != -ENOENT) {
+atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
backend_dev->rdma_dev_res);
}
} while (cqe_ctx_id != -ENOENT);
}
-static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
+static int rdma_poll_cq(RdmaBackendDev *backend_dev,
+RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
{
int i, ne, total_ne = 0;
BackendCtx *bctx;
@@ -113,6 +117,8 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
struct ibv_cq *ibcq)
comp_handler(bctx->up_ctx, &wc[i]);
+rdma_protected_gslist_remove_int32(&bctx->backend_qp->cqe_ctx_list,
+ wc[i].wr_id);
rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
g_free(bctx);
}
@@ -175,14 +181,12 @@ static void *comp_handler_thread(void *arg)
}
backend_dev->rdma_dev_res->stats.poll_cq_from_bk++;
-rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq);
+rdma_poll_cq(backend_dev, backend_dev->rdma_dev_res, ev_cq);
ibv_ack_cq_events(ev_cq, 1);
}
}
-/* TODO: Post cqe for all remaining buffs that were posted */
-
backend_dev->comp_thread.is_running = false;
qemu_thread_exit(0);
@@ -311,7 +315,7 @@ void rdma_backend_poll_cq(RdmaDeviceResources
*rdma_dev_res, RdmaBackendCQ *cq)
int polled;
rdma_dev_res->stats.poll_cq_from_guest++;
-polled = rdma_poll_cq(rdma_dev_res, cq->ibcq);
+polled = rdma_poll_cq(cq->backend_dev, rdma_dev_res, cq->ibcq);
if (!polled) {
rdma_dev_res->stats.poll_cq_from_guest_empty++;
}
@@ -501,6 +505,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
bctx = g_malloc0(sizeof(*bctx));
bctx->up_ctx = ctx;
+bctx->backend_qp = qp;
rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
if (unlikely(rc)) {
@@ -508,6 +513,8 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
goto err_free_bctx;
}
+rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
+
rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
&backend_dev->rdma_dev_res->stats.tx_len);
if (rc) {
@@ -616,6 +623,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
bctx = g_malloc0(sizeof(*bctx));
bctx->up_ctx = ctx;
+bctx->backend_qp = qp;
rc = rdma_rm_alloc_cqe_ctx(rdma_dev_res, &bctx_id, bctx);
if (unlikely(rc)) {
@@ -623,6 +631,8 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
goto err_free_bctx;
}
+rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
+
rc = build_host_sge_array(rdma_dev_res, new_sge, sge, num_sge,
&backend_dev->rdma_dev_res->stats.rx_bufs_len);
if (rc) {
@@ -762,6 +772,8 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
return -EIO;
}
+rdma_protected_gslist_init(&qp->cqe_ctx_list);
+
qp->ibpd = pd->ibpd;
/* TODO: Query QP to get max_inline_data and save it to be used in send */
@@ -919,11 +931,13 @@ int rdma_backend_query_qp(RdmaBackendQP *qp, struct
ibv_qp_attr *attr,
return ibv_quer
