Re: [Qemu-devel] [PATCH v1] hw/rdma: Add support for GID state changes for non-qmp frameworks
; > > -return ret; > +/* > + * We ignore return value since operation might have completed > + * successfully by the QMP consumer > + */ > +netlink_route_update(ifname, gid, RTM_NEWADDR); > + > +return 0; > } > > int rdma_backend_del_gid(RdmaBackendDev *backend_dev, const char *ifname, > @@ -1149,6 +1215,12 @@ int rdma_backend_del_gid(RdmaBackendDev *backend_dev, > const char *ifname, > gid->global.subnet_prefix, > gid->global.interface_id); > > +/* > + * We ignore return value since operation might have completed > + * successfully by the QMP consumer > + */ > +netlink_route_update(ifname, gid, RTM_DELADDR); > + > return 0; > } > > Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH] hw/rdma: Delete unused headers inclusion
On 5/5/19 1:51 PM, Yuval Shaia wrote: > This is a trivial cleanup patch. > > Signed-off-by: Yuval Shaia > --- > hw/rdma/rdma_backend.c | 7 --- > 1 file changed, 7 deletions(-) > > diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c > index d1660b6474..05f6b03221 100644 > --- a/hw/rdma/rdma_backend.c > +++ b/hw/rdma/rdma_backend.c > @@ -14,16 +14,9 @@ > */ > > #include "qemu/osdep.h" > -#include "sysemu/sysemu.h" > -#include "qapi/error.h" > -#include "qapi/qmp/qlist.h" > -#include "qapi/qmp/qnum.h" > #include "qapi/qapi-events-rdma.h" > > #include > -#include > -#include > -#include > > #include "contrib/rdmacm-mux/rdmacm-mux.h" > #include "trace.h" > Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v3 1/4] hw/rdma: Add SRQ support to backend layer
On 4/3/19 9:05 PM, Yuval Shaia wrote:
> On Wed, Apr 03, 2019 at 02:33:40PM +0300, Kamal Heib wrote:
>> Add the required functions and definitions to support shared receive
>> queues (SRQs) in the backend layer.
>>
>> Signed-off-by: Kamal Heib
>> ---
>> hw/rdma/rdma_backend.c | 116 +++-
>> hw/rdma/rdma_backend.h | 12
>> hw/rdma/rdma_backend_defs.h | 5 ++
>> hw/rdma/rdma_rm.c | 2 +
>> hw/rdma/rdma_rm_defs.h | 1 +
>> 5 files changed, 134 insertions(+), 2 deletions(-)
>>
>> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
>> index d1660b6474fa..04dfd63a573b 100644
>> --- a/hw/rdma/rdma_backend.c
>> +++ b/hw/rdma/rdma_backend.c
>> @@ -40,6 +40,7 @@ typedef struct BackendCtx {
>> void *up_ctx;
>> struct ibv_sge sge; /* Used to save MAD recv buffer */
>> RdmaBackendQP *backend_qp; /* To maintain recv buffers */
>> +RdmaBackendSRQ *backend_srq;
>> } BackendCtx;
>>
>> struct backend_umad {
>> @@ -99,6 +100,7 @@ static int rdma_poll_cq(RdmaDeviceResources
>> *rdma_dev_res, struct ibv_cq *ibcq)
>> int i, ne, total_ne = 0;
>> BackendCtx *bctx;
>> struct ibv_wc wc[2];
>> +RdmaProtectedGSList *cqe_ctx_list;
>>
>> qemu_mutex_lock(_dev_res->lock);
>> do {
>> @@ -116,8 +118,13 @@ static int rdma_poll_cq(RdmaDeviceResources
>> *rdma_dev_res, struct ibv_cq *ibcq)
>>
>> comp_handler(bctx->up_ctx, [i]);
>>
>> -
>> rdma_protected_gslist_remove_int32(>backend_qp->cqe_ctx_list,
>> - wc[i].wr_id);
>> +if (bctx->backend_qp) {
>> +cqe_ctx_list = >backend_qp->cqe_ctx_list;
>> +} else {
>> +cqe_ctx_list = >backend_srq->cqe_ctx_list;
>> +}
>> +
>> +rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
>> rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
>> g_free(bctx);
>> }
>> @@ -662,6 +669,60 @@ err_free_bctx:
>> g_free(bctx);
>> }
>>
>> +void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
>> +RdmaBackendSRQ *srq, struct ibv_sge *sge,
>> +uint32_t num_sge, void *ctx)
>> +{
>> +BackendCtx *bctx;
>> +struct ibv_sge new_sge[MAX_SGE];
>> +uint32_t bctx_id;
>> +int rc;
>> +struct ibv_recv_wr wr = {}, *bad_wr;
>> +
>> +bctx = g_malloc0(sizeof(*bctx));
>> +bctx->up_ctx = ctx;
>> +bctx->backend_srq = srq;
>> +
>> +rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, _id, bctx);
>> +if (unlikely(rc)) {
>> +complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
>> +goto err_free_bctx;
>> +}
>> +
>> +rdma_protected_gslist_append_int32(>cqe_ctx_list, bctx_id);
>> +
>> +rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge,
>> num_sge,
>> +
>> _dev->rdma_dev_res->stats.rx_bufs_len);
>> +if (rc) {
>> +complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
>> +goto err_dealloc_cqe_ctx;
>> +}
>> +
>> +wr.num_sge = num_sge;
>> +wr.sg_list = new_sge;
>> +wr.wr_id = bctx_id;
>> +rc = ibv_post_srq_recv(srq->ibsrq, , _wr);
>> +if (rc) {
>> +rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d,
>> errno=%d",
>> + srq->ibsrq->handle, rc, errno);
>> +complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
>> +goto err_dealloc_cqe_ctx;
>> +}
>> +
>> +atomic_inc(_dev->rdma_dev_res->stats.missing_cqe);
>> +backend_dev->rdma_dev_res->stats.rx_bufs++;
>> +backend_dev->rdma_dev_res->stats.rx_srq++;
>
> You should update function rdma_dump_device_counters with this new
> counter.
>
>> +
>> +return;
>> +
>> +err_dealloc_cqe_ctx:
>> +backend_dev->rdma_dev_res->stats.rx_bufs_err++;
>> +rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
>> +
>> +err_free_bctx:
>> +g_free(bctx);
>> +}
>> +
>> int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
>> {
>> p
[Qemu-devel] [PATCH v3 4/4] hw/pvrdma: Add support for SRQ
Implement the pvrdma device commands for supporting SRQ
Signed-off-by: Kamal Heib
---
hw/rdma/vmw/pvrdma_cmd.c| 147
hw/rdma/vmw/pvrdma_main.c | 16
hw/rdma/vmw/pvrdma_qp_ops.c | 46 ++-
hw/rdma/vmw/pvrdma_qp_ops.h | 1 +
4 files changed, 209 insertions(+), 1 deletion(-)
diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
index b931bb6dc9d4..8d70c0d23de4 100644
--- a/hw/rdma/vmw/pvrdma_cmd.c
+++ b/hw/rdma/vmw/pvrdma_cmd.c
@@ -609,6 +609,149 @@ static int destroy_uc(PVRDMADev *dev, union
pvrdma_cmd_req *req,
return 0;
}
+static int create_srq_ring(PCIDevice *pci_dev, PvrdmaRing **ring,
+ uint64_t pdir_dma, uint32_t max_wr,
+ uint32_t max_sge, uint32_t nchunks)
+{
+uint64_t *dir = NULL, *tbl = NULL;
+PvrdmaRing *r;
+int rc = -EINVAL;
+char ring_name[MAX_RING_NAME_SZ];
+uint32_t wqe_sz;
+
+if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) {
+rdma_error_report("Got invalid page count for SRQ ring: %d",
+ nchunks);
+return rc;
+}
+
+dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
+if (!dir) {
+rdma_error_report("Failed to map to SRQ page directory");
+goto out;
+}
+
+tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
+if (!tbl) {
+rdma_error_report("Failed to map to SRQ page table");
+goto out;
+}
+
+r = g_malloc(sizeof(*r));
+*ring = r;
+
+r->ring_state = (struct pvrdma_ring *)
+rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
+if (!r->ring_state) {
+rdma_error_report("Failed to map tp SRQ ring state");
+goto out_free_ring_mem;
+}
+
+wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) +
+ sizeof(struct pvrdma_sge) * max_sge - 1);
+sprintf(ring_name, "srq_ring_%" PRIx64, pdir_dma);
+rc = pvrdma_ring_init(r, ring_name, pci_dev, >ring_state[1], max_wr,
+ wqe_sz, (dma_addr_t *)[1], nchunks - 1);
+if (rc) {
+goto out_unmap_ring_state;
+}
+
+goto out;
+
+out_unmap_ring_state:
+rdma_pci_dma_unmap(pci_dev, r->ring_state, TARGET_PAGE_SIZE);
+
+out_free_ring_mem:
+g_free(r);
+
+out:
+rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
+rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
+
+return rc;
+}
+
+static void destroy_srq_ring(PvrdmaRing *ring)
+{
+pvrdma_ring_free(ring);
+rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE);
+g_free(ring);
+}
+
+static int create_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_create_srq *cmd = >create_srq;
+struct pvrdma_cmd_create_srq_resp *resp = >create_srq_resp;
+PvrdmaRing *ring = NULL;
+int rc;
+
+memset(resp, 0, sizeof(*resp));
+
+rc = create_srq_ring(PCI_DEVICE(dev), , cmd->pdir_dma,
+ cmd->attrs.max_wr, cmd->attrs.max_sge,
+ cmd->nchunks);
+if (rc) {
+return rc;
+}
+
+rc = rdma_rm_alloc_srq(>rdma_dev_res, cmd->pd_handle,
+ cmd->attrs.max_wr, cmd->attrs.max_sge,
+ cmd->attrs.srq_limit, >srqn, ring);
+if (rc) {
+destroy_srq_ring(ring);
+return rc;
+}
+
+return 0;
+}
+
+static int query_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_query_srq *cmd = >query_srq;
+struct pvrdma_cmd_query_srq_resp *resp = >query_srq_resp;
+
+memset(resp, 0, sizeof(*resp));
+
+return rdma_rm_query_srq(>rdma_dev_res, cmd->srq_handle,
+ (struct ibv_srq_attr *)>attrs);
+}
+
+static int modify_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_modify_srq *cmd = >modify_srq;
+
+/* Only support SRQ limit */
+if (!(cmd->attr_mask & IBV_SRQ_LIMIT) ||
+(cmd->attr_mask & IBV_SRQ_MAX_WR))
+return -EINVAL;
+
+return rdma_rm_modify_srq(>rdma_dev_res, cmd->srq_handle,
+ (struct ibv_srq_attr *)>attrs,
+ cmd->attr_mask);
+}
+
+static int destroy_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_destroy_srq *cmd = >destroy_srq;
+RdmaRmSRQ *srq;
+PvrdmaRing *ring;
+
+srq = rdma_rm_get_srq(>rdma_dev_res, cmd->srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+ring = (PvrdmaRing *)srq->opaque;
+destroy_srq_ring(ring);
+rdma_rm_dealloc_srq(>rdma_de
[Qemu-devel] [PATCH v3 3/4] hw/rdma: Modify create/destroy QP to support SRQ
Modify create/destroy QP to support shared receive queue and rearrange
the destroy_qp() code to avoid touching the QP after calling
rdma_rm_dealloc_qp().
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 9 --
hw/rdma/rdma_backend.h | 6 ++--
hw/rdma/rdma_rm.c| 22 +--
hw/rdma/rdma_rm.h| 3 +-
hw/rdma/rdma_rm_defs.h | 1 +
hw/rdma/vmw/pvrdma_cmd.c | 59
6 files changed, 67 insertions(+), 33 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index 04dfd63a573b..cf34874e9d2f 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -794,9 +794,9 @@ void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
- RdmaBackendCQ *rcq, uint32_t max_send_wr,
- uint32_t max_recv_wr, uint32_t max_send_sge,
- uint32_t max_recv_sge)
+ RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
+ uint32_t max_send_wr, uint32_t max_recv_wr,
+ uint32_t max_send_sge, uint32_t max_recv_sge)
{
struct ibv_qp_init_attr attr = {};
@@ -824,6 +824,9 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
attr.cap.max_recv_wr = max_recv_wr;
attr.cap.max_send_sge = max_send_sge;
attr.cap.max_recv_sge = max_recv_sge;
+if (srq) {
+attr.srq = srq->ibsrq;
+}
qp->ibqp = ibv_create_qp(pd->ibpd, );
if (!qp->ibqp) {
diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h
index cad7956d98e8..7c1a19a2b5ff 100644
--- a/hw/rdma/rdma_backend.h
+++ b/hw/rdma/rdma_backend.h
@@ -89,9 +89,9 @@ void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res,
RdmaBackendCQ *cq);
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
- RdmaBackendCQ *rcq, uint32_t max_send_wr,
- uint32_t max_recv_wr, uint32_t max_send_sge,
- uint32_t max_recv_sge);
+ RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
+ uint32_t max_send_wr, uint32_t max_recv_wr,
+ uint32_t max_send_sge, uint32_t max_recv_sge);
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey);
int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
index c0bb27cb6b90..96279e8d6561 100644
--- a/hw/rdma/rdma_rm.c
+++ b/hw/rdma/rdma_rm.c
@@ -386,12 +386,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
uint8_t qp_type, uint32_t max_send_wr,
uint32_t max_send_sge, uint32_t send_cq_handle,
uint32_t max_recv_wr, uint32_t max_recv_sge,
- uint32_t recv_cq_handle, void *opaque, uint32_t *qpn)
+ uint32_t recv_cq_handle, void *opaque, uint32_t *qpn,
+ uint8_t is_srq, uint32_t srq_handle)
{
int rc;
RdmaRmQP *qp;
RdmaRmCQ *scq, *rcq;
RdmaRmPD *pd;
+RdmaRmSRQ *srq = NULL;
uint32_t rm_qpn;
pd = rdma_rm_get_pd(dev_res, pd_handle);
@@ -408,6 +410,16 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
return -EINVAL;
}
+if (is_srq) {
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+rdma_error_report("Invalid srqn %d", srq_handle);
+return -EINVAL;
+}
+
+srq->recv_cq_handle = recv_cq_handle;
+}
+
if (qp_type == IBV_QPT_GSI) {
scq->notify = CNT_SET;
rcq->notify = CNT_SET;
@@ -424,10 +436,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
qp->send_cq_handle = send_cq_handle;
qp->recv_cq_handle = recv_cq_handle;
qp->opaque = opaque;
+qp->is_srq = is_srq;
rc = rdma_backend_create_qp(>backend_qp, qp_type, >backend_pd,
->backend_cq, >backend_cq,
max_send_wr,
-max_recv_wr, max_send_sge, max_recv_sge);
+>backend_cq, >backend_cq,
+is_srq ? >backend_srq : NULL,
+max_send_wr, max_recv_wr, max_send_sge,
+max_recv_sge);
+
if (rc) {
rc = -EIO;
goto out_dealloc_qp;
diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
index e88ab95e264b..e8639909cd34 100644
--- a/hw/rdma/rdma_rm.h
+++ b/hw/rdma/rdma_rm.h
@@ -53,7 +53,8 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev
[Qemu-devel] [PATCH 2/4] hw/rdma: Add support for managing SRQ resource
Adding the required functions and definitions for support managing the
shared receive queues (SRQs).
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_rm.c | 93 ++
hw/rdma/rdma_rm.h | 10 +
hw/rdma/rdma_rm_defs.h | 8
3 files changed, 111 insertions(+)
diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
index b683506b8616..c4fb140dcd96 100644
--- a/hw/rdma/rdma_rm.c
+++ b/hw/rdma/rdma_rm.c
@@ -544,6 +544,96 @@ void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res,
uint32_t qp_handle)
rdma_res_tbl_dealloc(_res->qp_tbl, qp->qpn);
}
+RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
+{
+return rdma_res_tbl_get(_res->srq_tbl, srq_handle);
+}
+
+int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
+ uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
+ uint32_t *srq_handle, void *opaque)
+{
+RdmaRmSRQ *srq;
+RdmaRmPD *pd;
+int rc;
+
+pd = rdma_rm_get_pd(dev_res, pd_handle);
+if (!pd) {
+return -EINVAL;
+}
+
+srq = rdma_res_tbl_alloc(_res->srq_tbl, srq_handle);
+if (!srq) {
+return -ENOMEM;
+}
+
+rc = rdma_backend_create_srq(>backend_srq, >backend_pd,
+ max_wr, max_sge, srq_limit);
+if (rc) {
+rc = -EIO;
+goto out_dealloc_srq;
+}
+
+srq->opaque = opaque;
+
+return 0;
+
+out_dealloc_srq:
+rdma_res_tbl_dealloc(_res->srq_tbl, *srq_handle);
+
+return rc;
+}
+
+int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+return rdma_backend_query_srq(>backend_srq, srq_attr);
+}
+
+int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr, int srq_attr_mask)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+if ((srq_attr_mask & IBV_SRQ_LIMIT) &&
+(srq_attr->srq_limit == 0)) {
+return -EINVAL;
+}
+
+if ((srq_attr_mask & IBV_SRQ_MAX_WR) &&
+(srq_attr->max_wr == 0)) {
+return -EINVAL;
+}
+
+return rdma_backend_modify_srq(>backend_srq, srq_attr,
+ srq_attr_mask);
+}
+
+void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return;
+}
+
+rdma_backend_destroy_srq(>backend_srq, dev_res);
+rdma_res_tbl_dealloc(_res->srq_tbl, srq_handle);
+}
+
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id)
{
void **cqe_ctx;
@@ -673,6 +763,8 @@ int rdma_rm_init(RdmaDeviceResources *dev_res, struct
ibv_device_attr *dev_attr)
res_tbl_init("CQE_CTX", _res->cqe_ctx_tbl, dev_attr->max_qp *
dev_attr->max_qp_wr, sizeof(void *));
res_tbl_init("UC", _res->uc_tbl, MAX_UCS, sizeof(RdmaRmUC));
+res_tbl_init("SRQ", _res->srq_tbl, dev_attr->max_srq,
+ sizeof(RdmaRmSRQ));
init_ports(dev_res);
@@ -691,6 +783,7 @@ void rdma_rm_fini(RdmaDeviceResources *dev_res,
RdmaBackendDev *backend_dev,
fini_ports(dev_res, backend_dev, ifname);
+res_tbl_free(_res->srq_tbl);
res_tbl_free(_res->uc_tbl);
res_tbl_free(_res->cqe_ctx_tbl);
res_tbl_free(_res->qp_tbl);
diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
index 4f03f9b8c5f1..e88ab95e264b 100644
--- a/hw/rdma/rdma_rm.h
+++ b/hw/rdma/rdma_rm.h
@@ -65,6 +65,16 @@ int rdma_rm_query_qp(RdmaDeviceResources *dev_res,
RdmaBackendDev *backend_dev,
int attr_mask, struct ibv_qp_init_attr *init_attr);
void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle);
+RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle);
+int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
+ uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
+ uint32_t *srq_handle, void *opaque);
+int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr);
+int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr, int srq_attr_mask);
+void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle);
+
int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id,
void *ctx);
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
diff --gi
[Qemu-devel] [PATCH v3 0/4] pvrdma: Add support for SRQ
This series implements the SRQ (Shared Receive Queue) for the pvrdma device, It also includes all the needed functions and definitions for support SRQ in the backend and resource management layers. Changes from v2->3: - Patch #1: -- Fix commit message. -- Remove initialization of backend_qp from rdma_backend_post_srq_recv(). -- Add rx_srq counter. - Patch #2: -- Add checks for srq attrs. - Patch #3: -- Move initialization of recv_cq_handle to be under is_srq. -- Rearrange destroy_qp() to avoid use after free. - Patch #4: -- Avoid use after free. -- Fix indentation. Changes from v1->v2: - Handle checkpatch.pl warnings. Kamal Heib (4): hw/rdma: Add SRQ support to backend layer hw/rdma: Add support for managing SRQ resource hw/rdma: Modify create/destroy QP to support SRQ hw/pvrdma: Add support for SRQ hw/rdma/rdma_backend.c | 125 +- hw/rdma/rdma_backend.h | 18 +++- hw/rdma/rdma_backend_defs.h | 5 + hw/rdma/rdma_rm.c | 117 +++- hw/rdma/rdma_rm.h | 13 ++- hw/rdma/rdma_rm_defs.h | 10 ++ hw/rdma/vmw/pvrdma_cmd.c| 206 hw/rdma/vmw/pvrdma_main.c | 16 +++ hw/rdma/vmw/pvrdma_qp_ops.c | 46 +++- hw/rdma/vmw/pvrdma_qp_ops.h | 1 + 10 files changed, 521 insertions(+), 36 deletions(-) -- 2.20.1
[Qemu-devel] [PATCH v3 1/4] hw/rdma: Add SRQ support to backend layer
Add the required functions and definitions to support shared receive
queues (SRQs) in the backend layer.
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 116 +++-
hw/rdma/rdma_backend.h | 12
hw/rdma/rdma_backend_defs.h | 5 ++
hw/rdma/rdma_rm.c | 2 +
hw/rdma/rdma_rm_defs.h | 1 +
5 files changed, 134 insertions(+), 2 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index d1660b6474fa..04dfd63a573b 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -40,6 +40,7 @@ typedef struct BackendCtx {
void *up_ctx;
struct ibv_sge sge; /* Used to save MAD recv buffer */
RdmaBackendQP *backend_qp; /* To maintain recv buffers */
+RdmaBackendSRQ *backend_srq;
} BackendCtx;
struct backend_umad {
@@ -99,6 +100,7 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
struct ibv_cq *ibcq)
int i, ne, total_ne = 0;
BackendCtx *bctx;
struct ibv_wc wc[2];
+RdmaProtectedGSList *cqe_ctx_list;
qemu_mutex_lock(_dev_res->lock);
do {
@@ -116,8 +118,13 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
struct ibv_cq *ibcq)
comp_handler(bctx->up_ctx, [i]);
-rdma_protected_gslist_remove_int32(>backend_qp->cqe_ctx_list,
- wc[i].wr_id);
+if (bctx->backend_qp) {
+cqe_ctx_list = >backend_qp->cqe_ctx_list;
+} else {
+cqe_ctx_list = >backend_srq->cqe_ctx_list;
+}
+
+rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
g_free(bctx);
}
@@ -662,6 +669,60 @@ err_free_bctx:
g_free(bctx);
}
+void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
+RdmaBackendSRQ *srq, struct ibv_sge *sge,
+uint32_t num_sge, void *ctx)
+{
+BackendCtx *bctx;
+struct ibv_sge new_sge[MAX_SGE];
+uint32_t bctx_id;
+int rc;
+struct ibv_recv_wr wr = {}, *bad_wr;
+
+bctx = g_malloc0(sizeof(*bctx));
+bctx->up_ctx = ctx;
+bctx->backend_srq = srq;
+
+rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, _id, bctx);
+if (unlikely(rc)) {
+complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
+goto err_free_bctx;
+}
+
+rdma_protected_gslist_append_int32(>cqe_ctx_list, bctx_id);
+
+rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
+ _dev->rdma_dev_res->stats.rx_bufs_len);
+if (rc) {
+complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
+goto err_dealloc_cqe_ctx;
+}
+
+wr.num_sge = num_sge;
+wr.sg_list = new_sge;
+wr.wr_id = bctx_id;
+rc = ibv_post_srq_recv(srq->ibsrq, , _wr);
+if (rc) {
+rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d, errno=%d",
+ srq->ibsrq->handle, rc, errno);
+complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
+goto err_dealloc_cqe_ctx;
+}
+
+atomic_inc(_dev->rdma_dev_res->stats.missing_cqe);
+backend_dev->rdma_dev_res->stats.rx_bufs++;
+backend_dev->rdma_dev_res->stats.rx_srq++;
+
+return;
+
+err_dealloc_cqe_ctx:
+backend_dev->rdma_dev_res->stats.rx_bufs_err++;
+rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
+
+err_free_bctx:
+g_free(bctx);
+}
+
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
{
pd->ibpd = ibv_alloc_pd(backend_dev->context);
@@ -938,6 +999,55 @@ void rdma_backend_destroy_qp(RdmaBackendQP *qp,
RdmaDeviceResources *dev_res)
rdma_protected_gslist_destroy(>cqe_ctx_list);
}
+int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
+uint32_t max_wr, uint32_t max_sge,
+uint32_t srq_limit)
+{
+struct ibv_srq_init_attr srq_init_attr = {};
+
+srq_init_attr.attr.max_wr = max_wr;
+srq_init_attr.attr.max_sge = max_sge;
+srq_init_attr.attr.srq_limit = srq_limit;
+
+srq->ibsrq = ibv_create_srq(pd->ibpd, _init_attr);
+if (!srq->ibsrq) {
+rdma_error_report("ibv_create_srq failed, errno=%d", errno);
+return -EIO;
+}
+
+rdma_protected_gslist_init(>cqe_ctx_list);
+
+return 0;
+}
+
+int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr)
+{
+if (!srq->ibsrq) {
+return -EINVAL;
+}
+
+return ibv_query_srq(srq->ibsrq, srq_attr);
+}
+
+int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
+int srq_attr_mask)
+{
+if (!srq->ibsrq) {
+return -EINVAL;
+}
+
+return ibv_m
Re: [Qemu-devel] [PATCH v2 0/4] pvrdma: Add support for SRQ
On 3/27/19 6:18 PM, Yuval Shaia wrote: > On Tue, Mar 26, 2019 at 02:54:29PM +0200, Kamal Heib wrote: >> This series implements the SRQ (Shared Receive Queue) for the pvrdma >> device, It also includes all the needed functions and definitions for >> support SRQ in the backend and resource management layers. >> >> Changes from v1->v2: >> - Handle checkpatch.pl warnings. > > Hi Kamal, Hi Yuval, > I'm done with the review, please check my comments. > Thank you for reviewing the patch set, I'll address the comments in v3. > Thanks for doing it, this is an advance feature that i wanted to do for > long time. > I'm very happy to contribute to this project :-) Thanks, Kamal >> >> Kamal Heib (4): >> hw/rdma: Add SRQ support to backend layer >> hw/rdma: Add support for managing SRQ resource >> hw/rdma: Modify create/destroy QP to support SRQ >> hw/pvrdma: Add support for SRQ >> >> hw/rdma/rdma_backend.c | 125 +- >> hw/rdma/rdma_backend.h | 18 +++- >> hw/rdma/rdma_backend_defs.h | 5 + >> hw/rdma/rdma_rm.c | 106 +- >> hw/rdma/rdma_rm.h | 13 ++- >> hw/rdma/rdma_rm_defs.h | 9 ++ >> hw/rdma/vmw/pvrdma_cmd.c| 208 >> hw/rdma/vmw/pvrdma_main.c | 16 +++ >> hw/rdma/vmw/pvrdma_qp_ops.c | 46 +++- >> hw/rdma/vmw/pvrdma_qp_ops.h | 1 + >> 10 files changed, 513 insertions(+), 34 deletions(-) >> >> -- >> 2.20.1 >> >>
Re: [Qemu-devel] [PATCH v2 4/4] hw/pvrdma: Add support for SRQ
On 3/27/19 6:16 PM, Yuval Shaia wrote:
> On Tue, Mar 26, 2019 at 02:54:33PM +0200, Kamal Heib wrote:
>> Implement the pvrdma device commands for supporting SRQ
>>
>> Signed-off-by: Kamal Heib
>> ---
>> hw/rdma/vmw/pvrdma_cmd.c| 147
>> hw/rdma/vmw/pvrdma_main.c | 16
>> hw/rdma/vmw/pvrdma_qp_ops.c | 46 ++-
>> hw/rdma/vmw/pvrdma_qp_ops.h | 1 +
>> 4 files changed, 209 insertions(+), 1 deletion(-)
>>
>> diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
>> index 510062f05476..fb075048c90c 100644
>> --- a/hw/rdma/vmw/pvrdma_cmd.c
>> +++ b/hw/rdma/vmw/pvrdma_cmd.c
>> @@ -615,6 +615,149 @@ static int destroy_uc(PVRDMADev *dev, union
>> pvrdma_cmd_req *req,
>> return 0;
>> }
>>
>> +static int create_srq_ring(PCIDevice *pci_dev, PvrdmaRing **ring,
>> + uint64_t pdir_dma, uint32_t max_wr,
>> + uint32_t max_sge, uint32_t nchunks)
>> +{
>> +uint64_t *dir = NULL, *tbl = NULL;
>> +PvrdmaRing *r;
>> +int rc = -EINVAL;
>> +char ring_name[MAX_RING_NAME_SZ];
>> +uint32_t wqe_sz;
>> +
>> +if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) {
>> +rdma_error_report("Got invalid page count for SRQ ring: %d",
>> + nchunks);
>> +return rc;
>> +}
>> +
>> +dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
>> +if (!dir) {
>> +rdma_error_report("Failed to map to SRQ page directory");
>> +goto out;
>> +}
>> +
>> +tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
>> +if (!tbl) {
>> +rdma_error_report("Failed to map to SRQ page table");
>> +goto out;
>> +}
>> +
>> +r = g_malloc(sizeof(*r));
>> +*ring = r;
>> +
>> +r->ring_state = (struct pvrdma_ring *)
>> +rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
>> +if (!r->ring_state) {
>> +rdma_error_report("Failed to map tp SRQ ring state");
>> +goto out_free_ring_mem;
>> +}
>> +
>> +wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) +
>> + sizeof(struct pvrdma_sge) * max_sge - 1);
>> +sprintf(ring_name, "srq_ring_%" PRIx64, pdir_dma);
>> +rc = pvrdma_ring_init(r, ring_name, pci_dev, >ring_state[1], max_wr,
>> + wqe_sz, (dma_addr_t *)[1], nchunks - 1);
>> +if (rc) {
>> +goto out_unmap_ring_state;
>> +}
>> +
>> +goto out;
>> +
>> +out_unmap_ring_state:
>> +rdma_pci_dma_unmap(pci_dev, r->ring_state, TARGET_PAGE_SIZE);
>> +
>> +out_free_ring_mem:
>> +g_free(r);
>> +
>> +out:
>> +rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
>> +rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
>> +
>> +return rc;
>> +}
>> +
>> +static void destroy_srq_ring(PvrdmaRing *ring)
>> +{
>> +pvrdma_ring_free(ring);
>> +rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE);
>> +g_free(ring);
>> +}
>> +
>> +static int create_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
>> + union pvrdma_cmd_resp *rsp)
>> +{
>> +struct pvrdma_cmd_create_srq *cmd = >create_srq;
>> +struct pvrdma_cmd_create_srq_resp *resp = >create_srq_resp;
>> +PvrdmaRing *ring = NULL;
>> +int rc;
>> +
>> +memset(resp, 0, sizeof(*resp));
>> +
>> +rc = create_srq_ring(PCI_DEVICE(dev), , cmd->pdir_dma,
>> + cmd->attrs.max_wr, cmd->attrs.max_sge,
>> + cmd->nchunks);
>> +if (rc) {
>> +return rc;
>> +}
>> +
>> +rc = rdma_rm_alloc_srq(>rdma_dev_res, cmd->pd_handle,
>> + cmd->attrs.max_wr, cmd->attrs.max_sge,
>> + cmd->attrs.srq_limit, >srqn, ring);
>> +if (rc) {
>> +destroy_srq_ring(ring);
>> +return rc;
>> +}
>> +
>> +return 0;
>> +}
>> +
>> +static int query_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
>> + union pvrdma_cmd_resp *rsp)
>> +{
>> +struct pvrdma_cmd_query_srq *cmd = >query_srq;
>> +struct pvrdma_cmd_query_srq_resp
Re: [Qemu-devel] [PATCH v2 3/4] hw/rdma: Modify create/destroy QP to support SRQ
On 3/27/19 5:54 PM, Yuval Shaia wrote:
> On Tue, Mar 26, 2019 at 02:54:32PM +0200, Kamal Heib wrote:
>> Modify create/destroy QP to support shared receive queue.
>>
>> Signed-off-by: Kamal Heib
>> ---
>> hw/rdma/rdma_backend.c | 9 --
>> hw/rdma/rdma_backend.h | 6 ++--
>> hw/rdma/rdma_rm.c| 23 +--
>> hw/rdma/rdma_rm.h| 3 +-
>> hw/rdma/rdma_rm_defs.h | 1 +
>> hw/rdma/vmw/pvrdma_cmd.c | 61 ++--
>> 6 files changed, 72 insertions(+), 31 deletions(-)
>>
>> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
>> index 54419c8c58dd..8f1349c64dda 100644
>> --- a/hw/rdma/rdma_backend.c
>> +++ b/hw/rdma/rdma_backend.c
>> @@ -794,9 +794,9 @@ void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
>>
>> int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
>> RdmaBackendPD *pd, RdmaBackendCQ *scq,
>> - RdmaBackendCQ *rcq, uint32_t max_send_wr,
>> - uint32_t max_recv_wr, uint32_t max_send_sge,
>> - uint32_t max_recv_sge)
>> + RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
>> + uint32_t max_send_wr, uint32_t max_recv_wr,
>> + uint32_t max_send_sge, uint32_t max_recv_sge)
>> {
>> struct ibv_qp_init_attr attr = {};
>>
>> @@ -824,6 +824,9 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
>> qp_type,
>> attr.cap.max_recv_wr = max_recv_wr;
>> attr.cap.max_send_sge = max_send_sge;
>> attr.cap.max_recv_sge = max_recv_sge;
>> +if (srq) {
>> +attr.srq = srq->ibsrq;
>> +}
>>
>> qp->ibqp = ibv_create_qp(pd->ibpd, );
>> if (!qp->ibqp) {
>> diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h
>> index cad7956d98e8..7c1a19a2b5ff 100644
>> --- a/hw/rdma/rdma_backend.h
>> +++ b/hw/rdma/rdma_backend.h
>> @@ -89,9 +89,9 @@ void rdma_backend_poll_cq(RdmaDeviceResources
>> *rdma_dev_res, RdmaBackendCQ *cq);
>>
>> int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
>> RdmaBackendPD *pd, RdmaBackendCQ *scq,
>> - RdmaBackendCQ *rcq, uint32_t max_send_wr,
>> - uint32_t max_recv_wr, uint32_t max_send_sge,
>> - uint32_t max_recv_sge);
>> + RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
>> + uint32_t max_send_wr, uint32_t max_recv_wr,
>> + uint32_t max_send_sge, uint32_t max_recv_sge);
>> int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP
>> *qp,
>> uint8_t qp_type, uint32_t qkey);
>> int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP
>> *qp,
>> diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
>> index bc5873cb4c14..90870ee0e15e 100644
>> --- a/hw/rdma/rdma_rm.c
>> +++ b/hw/rdma/rdma_rm.c
>> @@ -384,12 +384,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
>> uint32_t pd_handle,
>> uint8_t qp_type, uint32_t max_send_wr,
>> uint32_t max_send_sge, uint32_t send_cq_handle,
>> uint32_t max_recv_wr, uint32_t max_recv_sge,
>> - uint32_t recv_cq_handle, void *opaque, uint32_t *qpn)
>> + uint32_t recv_cq_handle, void *opaque, uint32_t *qpn,
>> + uint8_t is_srq, uint32_t srq_handle)
>> {
>> int rc;
>> RdmaRmQP *qp;
>> RdmaRmCQ *scq, *rcq;
>> RdmaRmPD *pd;
>> +RdmaRmSRQ *srq = NULL;
>> uint32_t rm_qpn;
>>
>> pd = rdma_rm_get_pd(dev_res, pd_handle);
>> @@ -406,6 +408,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
>> uint32_t pd_handle,
>> return -EINVAL;
>> }
>>
>> +if (is_srq) {
>> +srq = rdma_rm_get_srq(dev_res, srq_handle);
>> +if (!srq) {
>> +rdma_error_report("Invalid srqn %d", srq_handle);
>> +return -EINVAL;
>> +}
>> +}
>> +
>
> [1]
>
>> if (qp_type == IBV_QPT_GSI) {
>> scq->notify = CNT_SET;
>> rcq->notify = CNT_SET;
>> @@ -422,10 +432,17 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
>> uint32_t pd_handle,
Re: [Qemu-devel] [PATCH v2 2/4] hw/rdma: Add support for managing SRQ resource
On 3/27/19 6:03 PM, Yuval Shaia wrote:
> On Tue, Mar 26, 2019 at 02:54:31PM +0200, Kamal Heib wrote:
>> Adding the required functions and definitions for support managing the
>> shared receive queues (SRQs).
>>
>> Signed-off-by: Kamal Heib
>> ---
>> hw/rdma/rdma_rm.c | 83 ++
>> hw/rdma/rdma_rm.h | 10 +
>> hw/rdma/rdma_rm_defs.h | 8
>> 3 files changed, 101 insertions(+)
>>
>> diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
>> index bac3b2f4a6c3..bc5873cb4c14 100644
>> --- a/hw/rdma/rdma_rm.c
>> +++ b/hw/rdma/rdma_rm.c
>> @@ -542,6 +542,86 @@ void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res,
>> uint32_t qp_handle)
>> rdma_res_tbl_dealloc(_res->qp_tbl, qp->qpn);
>> }
>>
>> +RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t
>> srq_handle)
>> +{
>> +return rdma_res_tbl_get(_res->srq_tbl, srq_handle);
>> +}
>> +
>> +int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
>> + uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
>> + uint32_t *srq_handle, void *opaque)
>> +{
>> +RdmaRmSRQ *srq;
>> +RdmaRmPD *pd;
>> +int rc;
>> +
>> +pd = rdma_rm_get_pd(dev_res, pd_handle);
>> +if (!pd) {
>> +return -EINVAL;
>> +}
>> +
>> +srq = rdma_res_tbl_alloc(_res->srq_tbl, srq_handle);
>> +if (!srq) {
>> +return -ENOMEM;
>> +}
>> +
>> +rc = rdma_backend_create_srq(>backend_srq, >backend_pd,
>> + max_wr, max_sge, srq_limit);
>> +if (rc) {
>> +rc = -EIO;
>> +goto out_dealloc_srq;
>> +}
>> +
>> +srq->opaque = opaque;
>> +
>> +return 0;
>> +
>> +out_dealloc_srq:
>> +rdma_res_tbl_dealloc(_res->srq_tbl, *srq_handle);
>> +
>> +return rc;
>> +}
>> +
>> +int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
>> + struct ibv_srq_attr *srq_attr)
>> +{
>> +RdmaRmSRQ *srq;
>> +
>> +srq = rdma_rm_get_srq(dev_res, srq_handle);
>> +if (!srq) {
>> +return -EINVAL;
>> +}
>> +
>> +return rdma_backend_query_srq(>backend_srq, srq_attr);
>> +}
>> +
>> +int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
>> + struct ibv_srq_attr *srq_attr, int srq_attr_mask)
>> +{
>> +RdmaRmSRQ *srq;
>> +
>> +srq = rdma_rm_get_srq(dev_res, srq_handle);
>> +if (!srq) {
>> +return -EINVAL;
>> +}
>> +
>> +return rdma_backend_modify_srq(>backend_srq, srq_attr,
>> + srq_attr_mask);
>
> Such a blind pass-through?? don't you want to make sure that for example
> max_sge is valid? I mean just for the sake of being fair to caller?
>
I agree, I'll fix it in v3.
>> +}
>> +
>> +void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
>> +{
>> +RdmaRmSRQ *srq;
>> +
>> +srq = rdma_rm_get_srq(dev_res, srq_handle);
>> +if (!srq) {
>> +return;
>> +}
>> +
>> +rdma_backend_destroy_srq(>backend_srq, dev_res);
>> +rdma_res_tbl_dealloc(_res->srq_tbl, srq_handle);
>> +}
>> +
>> void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id)
>> {
>> void **cqe_ctx;
>> @@ -671,6 +751,8 @@ int rdma_rm_init(RdmaDeviceResources *dev_res, struct
>> ibv_device_attr *dev_attr)
>> res_tbl_init("CQE_CTX", _res->cqe_ctx_tbl, dev_attr->max_qp *
>> dev_attr->max_qp_wr, sizeof(void *));
>> res_tbl_init("UC", _res->uc_tbl, MAX_UCS, sizeof(RdmaRmUC));
>> +res_tbl_init("SRQ", _res->srq_tbl, dev_attr->max_srq,
>> + sizeof(RdmaRmSRQ));
>>
>> init_ports(dev_res);
>>
>> @@ -689,6 +771,7 @@ void rdma_rm_fini(RdmaDeviceResources *dev_res,
>> RdmaBackendDev *backend_dev,
>>
>> fini_ports(dev_res, backend_dev, ifname);
>>
>> +res_tbl_free(_res->srq_tbl);
>> res_tbl_free(_res->uc_tbl);
>> res_tbl_free(_res->cqe_ctx_tbl);
>> res_tbl_free(_res->qp_tbl);
>> diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
>> index 4f03f9b8c5f1..e88a
Re: [Qemu-devel] [PATCH v2 1/4] hw/rdma: Add SRQ support to backend layer
On 3/27/19 8:44 AM, Yuval Shaia wrote:
> On Tue, Mar 26, 2019 at 02:54:30PM +0200, Kamal Heib wrote:
>> Add the required function and definitions for support shared receive
>
> s/function/functions
> s/for/to (but not sure about that though)
>
OK, I'll fix it in v3.
>> queues (SRQs) in the backend layer.
>>
>> Signed-off-by: Kamal Heib
>> ---
>> hw/rdma/rdma_backend.c | 116 +++-
>> hw/rdma/rdma_backend.h | 12
>> hw/rdma/rdma_backend_defs.h | 5 ++
>> 3 files changed, 131 insertions(+), 2 deletions(-)
>>
>> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
>> index d1660b6474fa..54419c8c58dd 100644
>> --- a/hw/rdma/rdma_backend.c
>> +++ b/hw/rdma/rdma_backend.c
>> @@ -40,6 +40,7 @@ typedef struct BackendCtx {
>> void *up_ctx;
>> struct ibv_sge sge; /* Used to save MAD recv buffer */
>> RdmaBackendQP *backend_qp; /* To maintain recv buffers */
>> +RdmaBackendSRQ *backend_srq;
>> } BackendCtx;
>>
>> struct backend_umad {
>> @@ -99,6 +100,7 @@ static int rdma_poll_cq(RdmaDeviceResources
>> *rdma_dev_res, struct ibv_cq *ibcq)
>> int i, ne, total_ne = 0;
>> BackendCtx *bctx;
>> struct ibv_wc wc[2];
>> +RdmaProtectedGSList *cqe_ctx_list;
>>
>> qemu_mutex_lock(_dev_res->lock);
>> do {
>> @@ -116,8 +118,13 @@ static int rdma_poll_cq(RdmaDeviceResources
>> *rdma_dev_res, struct ibv_cq *ibcq)
>>
>> comp_handler(bctx->up_ctx, [i]);
>>
>> -
>> rdma_protected_gslist_remove_int32(>backend_qp->cqe_ctx_list,
>> - wc[i].wr_id);
>> +if (bctx->backend_qp) {
>> +cqe_ctx_list = >backend_qp->cqe_ctx_list;
>> +} else {
>> +cqe_ctx_list = >backend_srq->cqe_ctx_list;
>> +}
>> +
>> +rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
>> rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
>> g_free(bctx);
>> }
>> @@ -662,6 +669,60 @@ err_free_bctx:
>> g_free(bctx);
>> }
>>
>> +void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
>> +RdmaBackendSRQ *srq, struct ibv_sge *sge,
>> +uint32_t num_sge, void *ctx)
>> +{
>> +BackendCtx *bctx;
>> +struct ibv_sge new_sge[MAX_SGE];
>> +uint32_t bctx_id;
>> +int rc;
>> +struct ibv_recv_wr wr = {}, *bad_wr;
>> +
>> +bctx = g_malloc0(sizeof(*bctx));
>> +bctx->up_ctx = ctx;
>> +bctx->backend_srq = srq;
>> +bctx->backend_qp = NULL;
>
> g_malloc0 takes care for this (otherwise expecting your touch in
> rdma_backend_post_recv and rdma_backend_post_send)
You are right, I'll fix it in v3.
>
>> +
>> +rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, _id, bctx);
>> +if (unlikely(rc)) {
>> +complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
>> +goto err_free_bctx;
>> +}
>> +
>> +rdma_protected_gslist_append_int32(>cqe_ctx_list, bctx_id);
>> +
>> +rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge,
>> num_sge,
>> +
>> _dev->rdma_dev_res->stats.rx_bufs_len);
>> +if (rc) {
>> +complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
>> +goto err_dealloc_cqe_ctx;
>> +}
>> +
>> +wr.num_sge = num_sge;
>> +wr.sg_list = new_sge;
>> +wr.wr_id = bctx_id;
>> +rc = ibv_post_srq_recv(srq->ibsrq, , _wr);
>> +if (rc) {
>> +rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d,
>> errno=%d",
>> + srq->ibsrq->handle, rc, errno);
>> +complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
>> +goto err_dealloc_cqe_ctx;
>> +}
>> +
>> +atomic_inc(_dev->rdma_dev_res->stats.missing_cqe);
>> +backend_dev->rdma_dev_res->stats.rx_bufs++;
>
> Suggesting to maintain a dedicated counter for srq_rx, what do you think?
>
Probably need to maintain both, I mean add a dedicated counter for srq_rx and
and maintain the existing rx_bufs, because the rx_buf is very generic.
>> +
>> +return;
>> +
>> +err_dealloc_cqe_ctx:
>> +backend_dev->rdma_dev_res->stats.r
[Qemu-devel] [PATCH v2 3/4] hw/rdma: Modify create/destroy QP to support SRQ
Modify create/destroy QP to support shared receive queue.
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 9 --
hw/rdma/rdma_backend.h | 6 ++--
hw/rdma/rdma_rm.c| 23 +--
hw/rdma/rdma_rm.h| 3 +-
hw/rdma/rdma_rm_defs.h | 1 +
hw/rdma/vmw/pvrdma_cmd.c | 61 ++--
6 files changed, 72 insertions(+), 31 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index 54419c8c58dd..8f1349c64dda 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -794,9 +794,9 @@ void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
- RdmaBackendCQ *rcq, uint32_t max_send_wr,
- uint32_t max_recv_wr, uint32_t max_send_sge,
- uint32_t max_recv_sge)
+ RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
+ uint32_t max_send_wr, uint32_t max_recv_wr,
+ uint32_t max_send_sge, uint32_t max_recv_sge)
{
struct ibv_qp_init_attr attr = {};
@@ -824,6 +824,9 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
attr.cap.max_recv_wr = max_recv_wr;
attr.cap.max_send_sge = max_send_sge;
attr.cap.max_recv_sge = max_recv_sge;
+if (srq) {
+attr.srq = srq->ibsrq;
+}
qp->ibqp = ibv_create_qp(pd->ibpd, );
if (!qp->ibqp) {
diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h
index cad7956d98e8..7c1a19a2b5ff 100644
--- a/hw/rdma/rdma_backend.h
+++ b/hw/rdma/rdma_backend.h
@@ -89,9 +89,9 @@ void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res,
RdmaBackendCQ *cq);
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
- RdmaBackendCQ *rcq, uint32_t max_send_wr,
- uint32_t max_recv_wr, uint32_t max_send_sge,
- uint32_t max_recv_sge);
+ RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
+ uint32_t max_send_wr, uint32_t max_recv_wr,
+ uint32_t max_send_sge, uint32_t max_recv_sge);
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey);
int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
index bc5873cb4c14..90870ee0e15e 100644
--- a/hw/rdma/rdma_rm.c
+++ b/hw/rdma/rdma_rm.c
@@ -384,12 +384,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
uint8_t qp_type, uint32_t max_send_wr,
uint32_t max_send_sge, uint32_t send_cq_handle,
uint32_t max_recv_wr, uint32_t max_recv_sge,
- uint32_t recv_cq_handle, void *opaque, uint32_t *qpn)
+ uint32_t recv_cq_handle, void *opaque, uint32_t *qpn,
+ uint8_t is_srq, uint32_t srq_handle)
{
int rc;
RdmaRmQP *qp;
RdmaRmCQ *scq, *rcq;
RdmaRmPD *pd;
+RdmaRmSRQ *srq = NULL;
uint32_t rm_qpn;
pd = rdma_rm_get_pd(dev_res, pd_handle);
@@ -406,6 +408,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
return -EINVAL;
}
+if (is_srq) {
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+rdma_error_report("Invalid srqn %d", srq_handle);
+return -EINVAL;
+}
+}
+
if (qp_type == IBV_QPT_GSI) {
scq->notify = CNT_SET;
rcq->notify = CNT_SET;
@@ -422,10 +432,17 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
qp->send_cq_handle = send_cq_handle;
qp->recv_cq_handle = recv_cq_handle;
qp->opaque = opaque;
+if (is_srq) {
+qp->is_srq = is_srq;
+srq->recv_cq_handle = recv_cq_handle;
+}
rc = rdma_backend_create_qp(>backend_qp, qp_type, >backend_pd,
->backend_cq, >backend_cq,
max_send_wr,
-max_recv_wr, max_send_sge, max_recv_sge);
+>backend_cq, >backend_cq,
+is_srq ? >backend_srq : NULL,
+max_send_wr, max_recv_wr, max_send_sge,
+max_recv_sge);
+
if (rc) {
rc = -EIO;
goto out_dealloc_qp;
diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
index e88ab95e264b..e8639909cd34 100644
--- a/hw/rdma/rdma_rm.h
+++ b/hw/rdma/rdma_rm.h
@@ -53,7 +53,8 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t
pd_handle,
uint8_t qp_type, u
[Qemu-devel] [PATCH v2 2/4] hw/rdma: Add support for managing SRQ resource
Adding the required functions and definitions for support managing the
shared receive queues (SRQs).
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_rm.c | 83 ++
hw/rdma/rdma_rm.h | 10 +
hw/rdma/rdma_rm_defs.h | 8
3 files changed, 101 insertions(+)
diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
index bac3b2f4a6c3..bc5873cb4c14 100644
--- a/hw/rdma/rdma_rm.c
+++ b/hw/rdma/rdma_rm.c
@@ -542,6 +542,86 @@ void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res,
uint32_t qp_handle)
rdma_res_tbl_dealloc(_res->qp_tbl, qp->qpn);
}
+RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
+{
+return rdma_res_tbl_get(_res->srq_tbl, srq_handle);
+}
+
+int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
+ uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
+ uint32_t *srq_handle, void *opaque)
+{
+RdmaRmSRQ *srq;
+RdmaRmPD *pd;
+int rc;
+
+pd = rdma_rm_get_pd(dev_res, pd_handle);
+if (!pd) {
+return -EINVAL;
+}
+
+srq = rdma_res_tbl_alloc(_res->srq_tbl, srq_handle);
+if (!srq) {
+return -ENOMEM;
+}
+
+rc = rdma_backend_create_srq(>backend_srq, >backend_pd,
+ max_wr, max_sge, srq_limit);
+if (rc) {
+rc = -EIO;
+goto out_dealloc_srq;
+}
+
+srq->opaque = opaque;
+
+return 0;
+
+out_dealloc_srq:
+rdma_res_tbl_dealloc(_res->srq_tbl, *srq_handle);
+
+return rc;
+}
+
+int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+return rdma_backend_query_srq(>backend_srq, srq_attr);
+}
+
+int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr, int srq_attr_mask)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+return rdma_backend_modify_srq(>backend_srq, srq_attr,
+ srq_attr_mask);
+}
+
+void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return;
+}
+
+rdma_backend_destroy_srq(>backend_srq, dev_res);
+rdma_res_tbl_dealloc(_res->srq_tbl, srq_handle);
+}
+
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id)
{
void **cqe_ctx;
@@ -671,6 +751,8 @@ int rdma_rm_init(RdmaDeviceResources *dev_res, struct
ibv_device_attr *dev_attr)
res_tbl_init("CQE_CTX", _res->cqe_ctx_tbl, dev_attr->max_qp *
dev_attr->max_qp_wr, sizeof(void *));
res_tbl_init("UC", _res->uc_tbl, MAX_UCS, sizeof(RdmaRmUC));
+res_tbl_init("SRQ", _res->srq_tbl, dev_attr->max_srq,
+ sizeof(RdmaRmSRQ));
init_ports(dev_res);
@@ -689,6 +771,7 @@ void rdma_rm_fini(RdmaDeviceResources *dev_res,
RdmaBackendDev *backend_dev,
fini_ports(dev_res, backend_dev, ifname);
+res_tbl_free(_res->srq_tbl);
res_tbl_free(_res->uc_tbl);
res_tbl_free(_res->cqe_ctx_tbl);
res_tbl_free(_res->qp_tbl);
diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
index 4f03f9b8c5f1..e88ab95e264b 100644
--- a/hw/rdma/rdma_rm.h
+++ b/hw/rdma/rdma_rm.h
@@ -65,6 +65,16 @@ int rdma_rm_query_qp(RdmaDeviceResources *dev_res,
RdmaBackendDev *backend_dev,
int attr_mask, struct ibv_qp_init_attr *init_attr);
void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle);
+RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle);
+int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
+ uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
+ uint32_t *srq_handle, void *opaque);
+int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr);
+int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr, int srq_attr_mask);
+void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle);
+
int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id,
void *ctx);
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
index c200d311de37..2a3a409d92a0 100644
--- a/hw/rdma/rdma_rm_defs.h
+++ b/hw/rdma/rdma_rm_defs.h
@@ -33,6 +33,7 @@
#define MAX_QP_RD_ATOM16
#define MAX_QP_INIT_RD_ATOM 16
#define MAX_AH
[Qemu-devel] [PATCH v2 4/4] hw/pvrdma: Add support for SRQ
Implement the pvrdma device commands for supporting SRQ
Signed-off-by: Kamal Heib
---
hw/rdma/vmw/pvrdma_cmd.c| 147
hw/rdma/vmw/pvrdma_main.c | 16
hw/rdma/vmw/pvrdma_qp_ops.c | 46 ++-
hw/rdma/vmw/pvrdma_qp_ops.h | 1 +
4 files changed, 209 insertions(+), 1 deletion(-)
diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
index 510062f05476..fb075048c90c 100644
--- a/hw/rdma/vmw/pvrdma_cmd.c
+++ b/hw/rdma/vmw/pvrdma_cmd.c
@@ -615,6 +615,149 @@ static int destroy_uc(PVRDMADev *dev, union
pvrdma_cmd_req *req,
return 0;
}
+static int create_srq_ring(PCIDevice *pci_dev, PvrdmaRing **ring,
+ uint64_t pdir_dma, uint32_t max_wr,
+ uint32_t max_sge, uint32_t nchunks)
+{
+uint64_t *dir = NULL, *tbl = NULL;
+PvrdmaRing *r;
+int rc = -EINVAL;
+char ring_name[MAX_RING_NAME_SZ];
+uint32_t wqe_sz;
+
+if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) {
+rdma_error_report("Got invalid page count for SRQ ring: %d",
+ nchunks);
+return rc;
+}
+
+dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
+if (!dir) {
+rdma_error_report("Failed to map to SRQ page directory");
+goto out;
+}
+
+tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
+if (!tbl) {
+rdma_error_report("Failed to map to SRQ page table");
+goto out;
+}
+
+r = g_malloc(sizeof(*r));
+*ring = r;
+
+r->ring_state = (struct pvrdma_ring *)
+rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
+if (!r->ring_state) {
+rdma_error_report("Failed to map tp SRQ ring state");
+goto out_free_ring_mem;
+}
+
+wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) +
+ sizeof(struct pvrdma_sge) * max_sge - 1);
+sprintf(ring_name, "srq_ring_%" PRIx64, pdir_dma);
+rc = pvrdma_ring_init(r, ring_name, pci_dev, >ring_state[1], max_wr,
+ wqe_sz, (dma_addr_t *)[1], nchunks - 1);
+if (rc) {
+goto out_unmap_ring_state;
+}
+
+goto out;
+
+out_unmap_ring_state:
+rdma_pci_dma_unmap(pci_dev, r->ring_state, TARGET_PAGE_SIZE);
+
+out_free_ring_mem:
+g_free(r);
+
+out:
+rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
+rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
+
+return rc;
+}
+
+static void destroy_srq_ring(PvrdmaRing *ring)
+{
+pvrdma_ring_free(ring);
+rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE);
+g_free(ring);
+}
+
+static int create_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_create_srq *cmd = >create_srq;
+struct pvrdma_cmd_create_srq_resp *resp = >create_srq_resp;
+PvrdmaRing *ring = NULL;
+int rc;
+
+memset(resp, 0, sizeof(*resp));
+
+rc = create_srq_ring(PCI_DEVICE(dev), , cmd->pdir_dma,
+ cmd->attrs.max_wr, cmd->attrs.max_sge,
+ cmd->nchunks);
+if (rc) {
+return rc;
+}
+
+rc = rdma_rm_alloc_srq(>rdma_dev_res, cmd->pd_handle,
+ cmd->attrs.max_wr, cmd->attrs.max_sge,
+ cmd->attrs.srq_limit, >srqn, ring);
+if (rc) {
+destroy_srq_ring(ring);
+return rc;
+}
+
+return 0;
+}
+
+static int query_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_query_srq *cmd = >query_srq;
+struct pvrdma_cmd_query_srq_resp *resp = >query_srq_resp;
+
+memset(resp, 0, sizeof(*resp));
+
+return rdma_rm_query_srq(>rdma_dev_res, cmd->srq_handle,
+ (struct ibv_srq_attr *)>attrs);
+}
+
+static int modify_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_modify_srq *cmd = >modify_srq;
+
+/* Only support SRQ limit */
+if (!(cmd->attr_mask & IBV_SRQ_LIMIT) ||
+(cmd->attr_mask & IBV_SRQ_MAX_WR))
+return -EINVAL;
+
+return rdma_rm_modify_srq(>rdma_dev_res, cmd->srq_handle,
+ (struct ibv_srq_attr *)>attrs,
+ cmd->attr_mask);
+}
+
+static int destroy_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_destroy_srq *cmd = >destroy_srq;
+RdmaRmSRQ *srq;
+PvrdmaRing *ring;
+
+srq = rdma_rm_get_srq(>rdma_dev_res, cmd->srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+rdma_rm_dealloc_srq(>rdma_dev_res, cmd->srq_handle);
+ring = (PvrdmaRing *)srq->opaque;
+
[Qemu-devel] [PATCH v2 1/4] hw/rdma: Add SRQ support to backend layer
Add the required function and definitions for support shared receive
queues (SRQs) in the backend layer.
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 116 +++-
hw/rdma/rdma_backend.h | 12
hw/rdma/rdma_backend_defs.h | 5 ++
3 files changed, 131 insertions(+), 2 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index d1660b6474fa..54419c8c58dd 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -40,6 +40,7 @@ typedef struct BackendCtx {
void *up_ctx;
struct ibv_sge sge; /* Used to save MAD recv buffer */
RdmaBackendQP *backend_qp; /* To maintain recv buffers */
+RdmaBackendSRQ *backend_srq;
} BackendCtx;
struct backend_umad {
@@ -99,6 +100,7 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
struct ibv_cq *ibcq)
int i, ne, total_ne = 0;
BackendCtx *bctx;
struct ibv_wc wc[2];
+RdmaProtectedGSList *cqe_ctx_list;
qemu_mutex_lock(_dev_res->lock);
do {
@@ -116,8 +118,13 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
struct ibv_cq *ibcq)
comp_handler(bctx->up_ctx, [i]);
-rdma_protected_gslist_remove_int32(>backend_qp->cqe_ctx_list,
- wc[i].wr_id);
+if (bctx->backend_qp) {
+cqe_ctx_list = >backend_qp->cqe_ctx_list;
+} else {
+cqe_ctx_list = >backend_srq->cqe_ctx_list;
+}
+
+rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
g_free(bctx);
}
@@ -662,6 +669,60 @@ err_free_bctx:
g_free(bctx);
}
+void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
+RdmaBackendSRQ *srq, struct ibv_sge *sge,
+uint32_t num_sge, void *ctx)
+{
+BackendCtx *bctx;
+struct ibv_sge new_sge[MAX_SGE];
+uint32_t bctx_id;
+int rc;
+struct ibv_recv_wr wr = {}, *bad_wr;
+
+bctx = g_malloc0(sizeof(*bctx));
+bctx->up_ctx = ctx;
+bctx->backend_srq = srq;
+bctx->backend_qp = NULL;
+
+rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, _id, bctx);
+if (unlikely(rc)) {
+complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
+goto err_free_bctx;
+}
+
+rdma_protected_gslist_append_int32(>cqe_ctx_list, bctx_id);
+
+rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
+ _dev->rdma_dev_res->stats.rx_bufs_len);
+if (rc) {
+complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
+goto err_dealloc_cqe_ctx;
+}
+
+wr.num_sge = num_sge;
+wr.sg_list = new_sge;
+wr.wr_id = bctx_id;
+rc = ibv_post_srq_recv(srq->ibsrq, , _wr);
+if (rc) {
+rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d, errno=%d",
+ srq->ibsrq->handle, rc, errno);
+complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
+goto err_dealloc_cqe_ctx;
+}
+
+atomic_inc(_dev->rdma_dev_res->stats.missing_cqe);
+backend_dev->rdma_dev_res->stats.rx_bufs++;
+
+return;
+
+err_dealloc_cqe_ctx:
+backend_dev->rdma_dev_res->stats.rx_bufs_err++;
+rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
+
+err_free_bctx:
+g_free(bctx);
+}
+
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
{
pd->ibpd = ibv_alloc_pd(backend_dev->context);
@@ -938,6 +999,55 @@ void rdma_backend_destroy_qp(RdmaBackendQP *qp,
RdmaDeviceResources *dev_res)
rdma_protected_gslist_destroy(>cqe_ctx_list);
}
+int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
+uint32_t max_wr, uint32_t max_sge,
+uint32_t srq_limit)
+{
+struct ibv_srq_init_attr srq_init_attr = {};
+
+srq_init_attr.attr.max_wr = max_wr;
+srq_init_attr.attr.max_sge = max_sge;
+srq_init_attr.attr.srq_limit = srq_limit;
+
+srq->ibsrq = ibv_create_srq(pd->ibpd, _init_attr);
+if (!srq->ibsrq) {
+rdma_error_report("ibv_create_srq failed, errno=%d", errno);
+return -EIO;
+}
+
+rdma_protected_gslist_init(>cqe_ctx_list);
+
+return 0;
+}
+
+int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr)
+{
+if (!srq->ibsrq) {
+return -EINVAL;
+}
+
+return ibv_query_srq(srq->ibsrq, srq_attr);
+}
+
+int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
+int srq_attr_mask)
+{
+if (!srq->ibsrq) {
+return -EINVAL;
+}
+
+return ibv_modify_srq(srq->ibsrq, srq_attr, srq_attr_mask);
+}
+
+void rdma_backend_destroy_srq(RdmaBackendSR
[Qemu-devel] [PATCH v2 0/4] pvrdma: Add support for SRQ
This series implements the SRQ (Shared Receive Queue) for the pvrdma device, It also includes all the needed functions and definitions for support SRQ in the backend and resource management layers. Changes from v1->v2: - Handle checkpatch.pl warnings. Kamal Heib (4): hw/rdma: Add SRQ support to backend layer hw/rdma: Add support for managing SRQ resource hw/rdma: Modify create/destroy QP to support SRQ hw/pvrdma: Add support for SRQ hw/rdma/rdma_backend.c | 125 +- hw/rdma/rdma_backend.h | 18 +++- hw/rdma/rdma_backend_defs.h | 5 + hw/rdma/rdma_rm.c | 106 +- hw/rdma/rdma_rm.h | 13 ++- hw/rdma/rdma_rm_defs.h | 9 ++ hw/rdma/vmw/pvrdma_cmd.c| 208 hw/rdma/vmw/pvrdma_main.c | 16 +++ hw/rdma/vmw/pvrdma_qp_ops.c | 46 +++- hw/rdma/vmw/pvrdma_qp_ops.h | 1 + 10 files changed, 513 insertions(+), 34 deletions(-) -- 2.20.1
[Qemu-devel] [PATCH 3/4] hw/rdma: Modify create/destroy QP to support SRQ
Modify create/destroy QP to support shared receive queue.
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 9 --
hw/rdma/rdma_backend.h | 6 ++--
hw/rdma/rdma_rm.c| 23 +--
hw/rdma/rdma_rm.h| 3 +-
hw/rdma/rdma_rm_defs.h | 1 +
hw/rdma/vmw/pvrdma_cmd.c | 61 ++--
6 files changed, 72 insertions(+), 31 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index d75872256207..60cf0ddb3c2a 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -793,9 +793,9 @@ void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
- RdmaBackendCQ *rcq, uint32_t max_send_wr,
- uint32_t max_recv_wr, uint32_t max_send_sge,
- uint32_t max_recv_sge)
+ RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
+ uint32_t max_send_wr, uint32_t max_recv_wr,
+ uint32_t max_send_sge, uint32_t max_recv_sge)
{
struct ibv_qp_init_attr attr = {};
@@ -823,6 +823,9 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
attr.cap.max_recv_wr = max_recv_wr;
attr.cap.max_send_sge = max_send_sge;
attr.cap.max_recv_sge = max_recv_sge;
+if (srq) {
+attr.srq = srq->ibsrq;
+}
qp->ibqp = ibv_create_qp(pd->ibpd, );
if (!qp->ibqp) {
diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h
index cad7956d98e8..7c1a19a2b5ff 100644
--- a/hw/rdma/rdma_backend.h
+++ b/hw/rdma/rdma_backend.h
@@ -89,9 +89,9 @@ void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res,
RdmaBackendCQ *cq);
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
- RdmaBackendCQ *rcq, uint32_t max_send_wr,
- uint32_t max_recv_wr, uint32_t max_send_sge,
- uint32_t max_recv_sge);
+ RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
+ uint32_t max_send_wr, uint32_t max_recv_wr,
+ uint32_t max_send_sge, uint32_t max_recv_sge);
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey);
int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
index bc5873cb4c14..86d69779dda8 100644
--- a/hw/rdma/rdma_rm.c
+++ b/hw/rdma/rdma_rm.c
@@ -384,12 +384,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
uint8_t qp_type, uint32_t max_send_wr,
uint32_t max_send_sge, uint32_t send_cq_handle,
uint32_t max_recv_wr, uint32_t max_recv_sge,
- uint32_t recv_cq_handle, void *opaque, uint32_t *qpn)
+ uint32_t recv_cq_handle, void *opaque, uint32_t *qpn,
+ uint8_t is_srq, uint32_t srq_handle)
{
int rc;
RdmaRmQP *qp;
RdmaRmCQ *scq, *rcq;
RdmaRmPD *pd;
+RdmaRmSRQ *srq = NULL;
uint32_t rm_qpn;
pd = rdma_rm_get_pd(dev_res, pd_handle);
@@ -406,6 +408,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
return -EINVAL;
}
+if (is_srq) {
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+rdma_error_report("Invalid srqn %d", srq_handle);
+return -EINVAL;
+}
+}
+
if (qp_type == IBV_QPT_GSI) {
scq->notify = CNT_SET;
rcq->notify = CNT_SET;
@@ -422,10 +432,17 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res,
uint32_t pd_handle,
qp->send_cq_handle = send_cq_handle;
qp->recv_cq_handle = recv_cq_handle;
qp->opaque = opaque;
+if (is_srq) {
+ qp->is_srq = is_srq;
+ srq->recv_cq_handle = recv_cq_handle;
+}
rc = rdma_backend_create_qp(>backend_qp, qp_type, >backend_pd,
->backend_cq, >backend_cq,
max_send_wr,
-max_recv_wr, max_send_sge, max_recv_sge);
+>backend_cq, >backend_cq,
+is_srq ? >backend_srq : NULL,
+max_send_wr, max_recv_wr, max_send_sge,
+max_recv_sge);
+
if (rc) {
rc = -EIO;
goto out_dealloc_qp;
diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
index e88ab95e264b..e8639909cd34 100644
--- a/hw/rdma/rdma_rm.h
+++ b/hw/rdma/rdma_rm.h
@@ -53,7 +53,8 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t
pd_handle,
uint8_t qp_type, u
[Qemu-devel] [PATCH 0/4] pvrdma: Add support for SRQ
This series implements the SRQ (Shared Receive Queue) for the pvrdma device, It also includes all the needed functions and definitions for support SRQ in the backend and resource management layers. Kamal Heib (4): hw/rdma: Add SRQ support to backend layer hw/rdma: Add support for managing SRQ resource hw/rdma: Modify create/destroy QP to support SRQ hw/pvrdma: Add support for SRQ hw/rdma/rdma_backend.c | 124 - hw/rdma/rdma_backend.h | 18 +++- hw/rdma/rdma_backend_defs.h | 5 + hw/rdma/rdma_rm.c | 106 +- hw/rdma/rdma_rm.h | 13 ++- hw/rdma/rdma_rm_defs.h | 9 ++ hw/rdma/vmw/pvrdma_cmd.c| 208 hw/rdma/vmw/pvrdma_main.c | 16 +++ hw/rdma/vmw/pvrdma_qp_ops.c | 46 +++- hw/rdma/vmw/pvrdma_qp_ops.h | 1 + 10 files changed, 512 insertions(+), 34 deletions(-) -- 2.20.1
[Qemu-devel] [PATCH 1/4] hw/rdma: Add SRQ support to backend layer
Add the required function and definitions for support shared receive
queues (SRQs) in the backend layer.
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 115 +++-
hw/rdma/rdma_backend.h | 12
hw/rdma/rdma_backend_defs.h | 5 ++
3 files changed, 130 insertions(+), 2 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index d1660b6474fa..d75872256207 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -40,6 +40,7 @@ typedef struct BackendCtx {
void *up_ctx;
struct ibv_sge sge; /* Used to save MAD recv buffer */
RdmaBackendQP *backend_qp; /* To maintain recv buffers */
+RdmaBackendSRQ *backend_srq;
} BackendCtx;
struct backend_umad {
@@ -116,8 +117,13 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
struct ibv_cq *ibcq)
comp_handler(bctx->up_ctx, [i]);
-rdma_protected_gslist_remove_int32(>backend_qp->cqe_ctx_list,
- wc[i].wr_id);
+if (bctx->backend_qp) {
+
rdma_protected_gslist_remove_int32(>backend_qp->cqe_ctx_list,
+ wc[i].wr_id);
+} else {
+
rdma_protected_gslist_remove_int32(>backend_srq->cqe_ctx_list,
+ wc[i].wr_id);
+}
rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
g_free(bctx);
}
@@ -662,6 +668,60 @@ err_free_bctx:
g_free(bctx);
}
+void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
+RdmaBackendSRQ *srq, struct ibv_sge *sge,
+uint32_t num_sge, void *ctx)
+{
+BackendCtx *bctx;
+struct ibv_sge new_sge[MAX_SGE];
+uint32_t bctx_id;
+int rc;
+struct ibv_recv_wr wr = {}, *bad_wr;
+
+bctx = g_malloc0(sizeof(*bctx));
+bctx->up_ctx = ctx;
+bctx->backend_srq = srq;
+bctx->backend_qp = NULL;
+
+rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, _id, bctx);
+if (unlikely(rc)) {
+complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
+goto err_free_bctx;
+}
+
+rdma_protected_gslist_append_int32(>cqe_ctx_list, bctx_id);
+
+rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
+ _dev->rdma_dev_res->stats.rx_bufs_len);
+if (rc) {
+complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
+goto err_dealloc_cqe_ctx;
+}
+
+wr.num_sge = num_sge;
+wr.sg_list = new_sge;
+wr.wr_id = bctx_id;
+rc = ibv_post_srq_recv(srq->ibsrq, , _wr);
+if (rc) {
+rdma_error_report("ibv_post_srq_recv failed, srqn=0x%x, rc=%d,
errno=%d",
+ srq->ibsrq->handle, rc, errno);
+complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
+goto err_dealloc_cqe_ctx;
+}
+
+atomic_inc(_dev->rdma_dev_res->stats.missing_cqe);
+backend_dev->rdma_dev_res->stats.rx_bufs++;
+
+return;
+
+err_dealloc_cqe_ctx:
+backend_dev->rdma_dev_res->stats.rx_bufs_err++;
+rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
+
+err_free_bctx:
+g_free(bctx);
+}
+
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
{
pd->ibpd = ibv_alloc_pd(backend_dev->context);
@@ -938,6 +998,55 @@ void rdma_backend_destroy_qp(RdmaBackendQP *qp,
RdmaDeviceResources *dev_res)
rdma_protected_gslist_destroy(>cqe_ctx_list);
}
+int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
+uint32_t max_wr, uint32_t max_sge,
+uint32_t srq_limit)
+{
+struct ibv_srq_init_attr srq_init_attr = {};
+
+srq_init_attr.attr.max_wr = max_wr;
+srq_init_attr.attr.max_sge = max_sge;
+srq_init_attr.attr.srq_limit = srq_limit;
+
+srq->ibsrq = ibv_create_srq(pd->ibpd, _init_attr);
+if (!srq->ibsrq) {
+rdma_error_report("ibv_create_srq failed, errno=%d", errno);
+return -EIO;
+}
+
+rdma_protected_gslist_init(>cqe_ctx_list);
+
+return 0;
+}
+
+int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr)
+{
+if (!srq->ibsrq) {
+return -EINVAL;
+}
+
+return ibv_query_srq(srq->ibsrq, srq_attr);
+}
+
+int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
+int srq_attr_mask)
+{
+if (!srq->ibsrq) {
+return -EINVAL;
+}
+
+return ibv_modify_srq(srq->ibsrq, srq_attr, srq_attr_mask);
+}
+
+void rdma_backend_destroy_srq(RdmaBackendSRQ *srq, RdmaDeviceResources
*dev_res)
+{
+if (srq->ibsrq) {
+ibv_destroy_srq(srq->ibsrq);
+}
+g_slist_foreach(srq->cqe_ctx_list.list, free_cqe_ctx, dev_res);
[Qemu-devel] [PATCH 2/4] hw/rdma: Add support for managing SRQ resource
Adding the required functions and definitions for support managing the
shared receive queues (SRQs).
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_rm.c | 83 ++
hw/rdma/rdma_rm.h | 10 +
hw/rdma/rdma_rm_defs.h | 8
3 files changed, 101 insertions(+)
diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
index bac3b2f4a6c3..bc5873cb4c14 100644
--- a/hw/rdma/rdma_rm.c
+++ b/hw/rdma/rdma_rm.c
@@ -542,6 +542,86 @@ void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res,
uint32_t qp_handle)
rdma_res_tbl_dealloc(_res->qp_tbl, qp->qpn);
}
+RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
+{
+return rdma_res_tbl_get(_res->srq_tbl, srq_handle);
+}
+
+int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
+ uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
+ uint32_t *srq_handle, void *opaque)
+{
+RdmaRmSRQ *srq;
+RdmaRmPD *pd;
+int rc;
+
+pd = rdma_rm_get_pd(dev_res, pd_handle);
+if (!pd) {
+return -EINVAL;
+}
+
+srq = rdma_res_tbl_alloc(_res->srq_tbl, srq_handle);
+if (!srq) {
+return -ENOMEM;
+}
+
+rc = rdma_backend_create_srq(>backend_srq, >backend_pd,
+ max_wr, max_sge, srq_limit);
+if (rc) {
+rc = -EIO;
+goto out_dealloc_srq;
+}
+
+srq->opaque = opaque;
+
+return 0;
+
+out_dealloc_srq:
+rdma_res_tbl_dealloc(_res->srq_tbl, *srq_handle);
+
+return rc;
+}
+
+int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+return rdma_backend_query_srq(>backend_srq, srq_attr);
+}
+
+int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr, int srq_attr_mask)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+return rdma_backend_modify_srq(>backend_srq, srq_attr,
+ srq_attr_mask);
+}
+
+void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle)
+{
+RdmaRmSRQ *srq;
+
+srq = rdma_rm_get_srq(dev_res, srq_handle);
+if (!srq) {
+return;
+}
+
+rdma_backend_destroy_srq(>backend_srq, dev_res);
+rdma_res_tbl_dealloc(_res->srq_tbl, srq_handle);
+}
+
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id)
{
void **cqe_ctx;
@@ -671,6 +751,8 @@ int rdma_rm_init(RdmaDeviceResources *dev_res, struct
ibv_device_attr *dev_attr)
res_tbl_init("CQE_CTX", _res->cqe_ctx_tbl, dev_attr->max_qp *
dev_attr->max_qp_wr, sizeof(void *));
res_tbl_init("UC", _res->uc_tbl, MAX_UCS, sizeof(RdmaRmUC));
+res_tbl_init("SRQ", _res->srq_tbl, dev_attr->max_srq,
+ sizeof(RdmaRmSRQ));
init_ports(dev_res);
@@ -689,6 +771,7 @@ void rdma_rm_fini(RdmaDeviceResources *dev_res,
RdmaBackendDev *backend_dev,
fini_ports(dev_res, backend_dev, ifname);
+res_tbl_free(_res->srq_tbl);
res_tbl_free(_res->uc_tbl);
res_tbl_free(_res->cqe_ctx_tbl);
res_tbl_free(_res->qp_tbl);
diff --git a/hw/rdma/rdma_rm.h b/hw/rdma/rdma_rm.h
index 4f03f9b8c5f1..e88ab95e264b 100644
--- a/hw/rdma/rdma_rm.h
+++ b/hw/rdma/rdma_rm.h
@@ -65,6 +65,16 @@ int rdma_rm_query_qp(RdmaDeviceResources *dev_res,
RdmaBackendDev *backend_dev,
int attr_mask, struct ibv_qp_init_attr *init_attr);
void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle);
+RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle);
+int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle,
+ uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit,
+ uint32_t *srq_handle, void *opaque);
+int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr);
+int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle,
+ struct ibv_srq_attr *srq_attr, int srq_attr_mask);
+void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle);
+
int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id,
void *ctx);
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
index c200d311de37..2a3a409d92a0 100644
--- a/hw/rdma/rdma_rm_defs.h
+++ b/hw/rdma/rdma_rm_defs.h
@@ -33,6 +33,7 @@
#define MAX_QP_RD_ATOM16
#define MAX_QP_INIT_RD_ATOM 16
#define MAX_AH
[Qemu-devel] [PATCH 4/4] hw/pvrdma: Add support for SRQ
Implement the pvrdma device commands for supporting SRQ
Signed-off-by: Kamal Heib
---
hw/rdma/vmw/pvrdma_cmd.c| 147
hw/rdma/vmw/pvrdma_main.c | 16
hw/rdma/vmw/pvrdma_qp_ops.c | 46 ++-
hw/rdma/vmw/pvrdma_qp_ops.h | 1 +
4 files changed, 209 insertions(+), 1 deletion(-)
diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
index 510062f05476..fb075048c90c 100644
--- a/hw/rdma/vmw/pvrdma_cmd.c
+++ b/hw/rdma/vmw/pvrdma_cmd.c
@@ -615,6 +615,149 @@ static int destroy_uc(PVRDMADev *dev, union
pvrdma_cmd_req *req,
return 0;
}
+static int create_srq_ring(PCIDevice *pci_dev, PvrdmaRing **ring,
+ uint64_t pdir_dma, uint32_t max_wr,
+ uint32_t max_sge, uint32_t nchunks)
+{
+uint64_t *dir = NULL, *tbl = NULL;
+PvrdmaRing *r;
+int rc = -EINVAL;
+char ring_name[MAX_RING_NAME_SZ];
+uint32_t wqe_sz;
+
+if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) {
+rdma_error_report("Got invalid page count for SRQ ring: %d",
+ nchunks);
+return rc;
+}
+
+dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
+if (!dir) {
+rdma_error_report("Failed to map to SRQ page directory");
+goto out;
+}
+
+tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
+if (!tbl) {
+rdma_error_report("Failed to map to SRQ page table");
+goto out;
+}
+
+r = g_malloc(sizeof(*r));
+*ring = r;
+
+r->ring_state = (struct pvrdma_ring *)
+rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
+if (!r->ring_state) {
+rdma_error_report("Failed to map tp SRQ ring state");
+goto out_free_ring_mem;
+}
+
+wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) +
+ sizeof(struct pvrdma_sge) * max_sge - 1);
+sprintf(ring_name, "srq_ring_%" PRIx64, pdir_dma);
+rc = pvrdma_ring_init(r, ring_name, pci_dev, >ring_state[1], max_wr,
+ wqe_sz, (dma_addr_t *)[1], nchunks - 1);
+if (rc) {
+goto out_unmap_ring_state;
+}
+
+goto out;
+
+out_unmap_ring_state:
+rdma_pci_dma_unmap(pci_dev, r->ring_state, TARGET_PAGE_SIZE);
+
+out_free_ring_mem:
+g_free(r);
+
+out:
+rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE);
+rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE);
+
+return rc;
+}
+
+static void destroy_srq_ring(PvrdmaRing *ring)
+{
+pvrdma_ring_free(ring);
+rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE);
+g_free(ring);
+}
+
+static int create_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_create_srq *cmd = >create_srq;
+struct pvrdma_cmd_create_srq_resp *resp = >create_srq_resp;
+PvrdmaRing *ring = NULL;
+int rc;
+
+memset(resp, 0, sizeof(*resp));
+
+rc = create_srq_ring(PCI_DEVICE(dev), , cmd->pdir_dma,
+ cmd->attrs.max_wr, cmd->attrs.max_sge,
+ cmd->nchunks);
+if (rc) {
+return rc;
+}
+
+rc = rdma_rm_alloc_srq(>rdma_dev_res, cmd->pd_handle,
+ cmd->attrs.max_wr, cmd->attrs.max_sge,
+ cmd->attrs.srq_limit, >srqn, ring);
+if (rc) {
+destroy_srq_ring(ring);
+return rc;
+}
+
+return 0;
+}
+
+static int query_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_query_srq *cmd = >query_srq;
+struct pvrdma_cmd_query_srq_resp *resp = >query_srq_resp;
+
+memset(resp, 0, sizeof(*resp));
+
+return rdma_rm_query_srq(>rdma_dev_res, cmd->srq_handle,
+ (struct ibv_srq_attr *)>attrs);
+}
+
+static int modify_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_modify_srq *cmd = >modify_srq;
+
+/* Only support SRQ limit */
+if (!(cmd->attr_mask & IBV_SRQ_LIMIT) ||
+(cmd->attr_mask & IBV_SRQ_MAX_WR))
+return -EINVAL;
+
+return rdma_rm_modify_srq(>rdma_dev_res, cmd->srq_handle,
+ (struct ibv_srq_attr *)>attrs,
+ cmd->attr_mask);
+}
+
+static int destroy_srq(PVRDMADev *dev, union pvrdma_cmd_req *req,
+ union pvrdma_cmd_resp *rsp)
+{
+struct pvrdma_cmd_destroy_srq *cmd = >destroy_srq;
+RdmaRmSRQ *srq;
+PvrdmaRing *ring;
+
+srq = rdma_rm_get_srq(>rdma_dev_res, cmd->srq_handle);
+if (!srq) {
+return -EINVAL;
+}
+
+rdma_rm_dealloc_srq(>rdma_dev_res, cmd->srq_handle);
+ring = (PvrdmaRing *)srq->opaque;
+
[Qemu-devel] [PATCH v2 4/4] hw/pvrdma: Fix zero-initialization of resp in {query/modify}_qp
Make sure to zero-initialize only the pvrdma_cmd_query_qp_resp and not the whole pvrdma_cmd_resp for query_qp, in modify_qp the resp isn't used so remove it. Reviewed-by: Yuval Shaia Signed-off-by: Kamal Heib --- hw/rdma/vmw/pvrdma_cmd.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c index a0f2b13a2438..d1ca20a3f90d 100644 --- a/hw/rdma/vmw/pvrdma_cmd.c +++ b/hw/rdma/vmw/pvrdma_cmd.c @@ -488,8 +488,6 @@ static int modify_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, struct pvrdma_cmd_modify_qp *cmd = >modify_qp; int rc; -memset(rsp, 0, sizeof(*rsp)); - /* No need to verify sgid_index since it is u8 */ rc = rdma_rm_modify_qp(>rdma_dev_res, >backend_dev, @@ -512,7 +510,7 @@ static int query_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, struct ibv_qp_init_attr init_attr; int rc; -memset(rsp, 0, sizeof(*rsp)); +memset(resp, 0, sizeof(*resp)); rc = rdma_rm_query_qp(>rdma_dev_res, >backend_dev, cmd->qp_handle, (struct ibv_qp_attr *)>attrs, cmd->attr_mask, -- 2.20.1
[Qemu-devel] [PATCH v2 3/4] hw/rdma: Use {} instead of {0}
Initialize structs with {} instead of {0} to make sure that all code is
using the same convention.
Reviewed-by: Marcel Apfelbaum
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 18 +-
hw/rdma/vmw/pvrdma_cmd.c| 2 +-
hw/rdma/vmw/pvrdma_qp_ops.c | 2 +-
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index 78bafc13642a..55b633e451c1 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -57,7 +57,7 @@ static void dummy_comp_handler(void *ctx, struct ibv_wc *wc)
static inline void complete_work(enum ibv_wc_status status, uint32_t
vendor_err,
void *ctx)
{
-struct ibv_wc wc = {0};
+struct ibv_wc wc = {};
wc.status = status;
wc.vendor_err = vendor_err;
@@ -273,7 +273,7 @@ static void stop_backend_thread(RdmaBackendThread *thread)
static void start_comp_thread(RdmaBackendDev *backend_dev)
{
-char thread_name[THR_NAME_LEN] = {0};
+char thread_name[THR_NAME_LEN] = {};
stop_backend_thread(_dev->comp_thread);
@@ -483,7 +483,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
struct ibv_sge new_sge[MAX_SGE];
uint32_t bctx_id;
int rc;
-struct ibv_send_wr wr = {0}, *bad_wr;
+struct ibv_send_wr wr = {}, *bad_wr;
if (!qp->ibqp) { /* This field is not initialized for QP0 and QP1 */
if (qp_type == IBV_QPT_SMI) {
@@ -600,7 +600,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
struct ibv_sge new_sge[MAX_SGE];
uint32_t bctx_id;
int rc;
-struct ibv_recv_wr wr = {0}, *bad_wr;
+struct ibv_recv_wr wr = {}, *bad_wr;
if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1 */
if (qp_type == IBV_QPT_SMI) {
@@ -737,7 +737,7 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
uint32_t max_recv_wr, uint32_t max_send_sge,
uint32_t max_recv_sge)
{
-struct ibv_qp_init_attr attr = {0};
+struct ibv_qp_init_attr attr = {};
qp->ibqp = 0;
@@ -782,7 +782,7 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey)
{
-struct ibv_qp_attr attr = {0};
+struct ibv_qp_attr attr = {};
int rc, attr_mask;
attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
@@ -821,7 +821,7 @@ int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp,
union ibv_gid *dgid, uint32_t dqpn,
uint32_t rq_psn, uint32_t qkey, bool use_qkey)
{
-struct ibv_qp_attr attr = {0};
+struct ibv_qp_attr attr = {};
union ibv_gid ibv_gid = {
.global.interface_id = dgid->global.interface_id,
.global.subnet_prefix = dgid->global.subnet_prefix
@@ -880,7 +880,7 @@ int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp,
int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
uint32_t sq_psn, uint32_t qkey, bool use_qkey)
{
-struct ibv_qp_attr attr = {0};
+struct ibv_qp_attr attr = {};
int rc, attr_mask;
attr.qp_state = IBV_QPS_RTS;
@@ -1012,7 +1012,7 @@ static void process_incoming_mad_req(RdmaBackendDev
*backend_dev,
complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF,
bctx->up_ctx);
} else {
-struct ibv_wc wc = {0};
+struct ibv_wc wc = {};
memset(mad, 0, bctx->sge.length);
build_mad_hdr((struct ibv_grh *)mad,
(union ibv_gid *)>umad.hdr.addr.gid, >hdr.sgid,
diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
index 21a55e225a61..a0f2b13a2438 100644
--- a/hw/rdma/vmw/pvrdma_cmd.c
+++ b/hw/rdma/vmw/pvrdma_cmd.c
@@ -123,7 +123,7 @@ static int query_port(PVRDMADev *dev, union pvrdma_cmd_req
*req,
{
struct pvrdma_cmd_query_port *cmd = >query_port;
struct pvrdma_cmd_query_port_resp *resp = >query_port_resp;
-struct pvrdma_port_attr attrs = {0};
+struct pvrdma_port_attr attrs = {};
if (cmd->port_num > MAX_PORTS) {
return -EINVAL;
diff --git a/hw/rdma/vmw/pvrdma_qp_ops.c b/hw/rdma/vmw/pvrdma_qp_ops.c
index 508d8fca3c9b..5b9786efbe4b 100644
--- a/hw/rdma/vmw/pvrdma_qp_ops.c
+++ b/hw/rdma/vmw/pvrdma_qp_ops.c
@@ -114,7 +114,7 @@ static void pvrdma_qp_ops_comp_handler(void *ctx, struct
ibv_wc *wc)
static void complete_with_error(uint32_t vendor_err, void *ctx)
{
-struct ibv_wc wc = {0};
+struct ibv_wc wc = {};
wc.status = IBV_WC_GENERAL_ERR;
wc.vendor_err = vendor_err;
--
2.20.1
[Qemu-devel] [PATCH v2 2/4] hw/rdma: Remove unused parameter from rdma_poll_cq()
The 'rdma_dev_res' parameter is not used in rdma_poll_cq(), so remove it.
Reviewed-by: Yuval Shaia
Reviewed-by: Marcel Apfelbaum
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index 66185bd487b6..78bafc13642a 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -94,8 +94,7 @@ static void clean_recv_mads(RdmaBackendDev *backend_dev)
} while (cqe_ctx_id != -ENOENT);
}
-static int rdma_poll_cq(RdmaBackendDev *backend_dev,
-RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
+static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
{
int i, ne, total_ne = 0;
BackendCtx *bctx;
@@ -181,7 +180,7 @@ static void *comp_handler_thread(void *arg)
}
backend_dev->rdma_dev_res->stats.poll_cq_from_bk++;
-rdma_poll_cq(backend_dev, backend_dev->rdma_dev_res, ev_cq);
+rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq);
ibv_ack_cq_events(ev_cq, 1);
}
@@ -315,7 +314,7 @@ void rdma_backend_poll_cq(RdmaDeviceResources
*rdma_dev_res, RdmaBackendCQ *cq)
int polled;
rdma_dev_res->stats.poll_cq_from_guest++;
-polled = rdma_poll_cq(cq->backend_dev, rdma_dev_res, cq->ibcq);
+polled = rdma_poll_cq(rdma_dev_res, cq->ibcq);
if (!polled) {
rdma_dev_res->stats.poll_cq_from_guest_empty++;
}
--
2.20.1
[Qemu-devel] [PATCH v2 0/4] pvrdma: misc fixes
The following patch set includes few misc fixes for the pvrdma driver.
Note: this patch set depends on the latest patch set from Yuval.
Changes from v1-v2:
- Update the commit message of patch #3.
Kamal Heib (4):
hw/rdma: Fix broken paths to docs/devel/tracing.txt
hw/rdma: Remove unused parameter from rdma_poll_cq()
hw/rdma: Use {} instead of {0}
hw/pvrdma: Fix zero-initialization of resp in {query/modify}_qp
hw/rdma/rdma_backend.c | 25 -
hw/rdma/trace-events| 2 +-
hw/rdma/vmw/pvrdma_cmd.c| 6 ++
hw/rdma/vmw/pvrdma_qp_ops.c | 2 +-
hw/rdma/vmw/trace-events| 2 +-
5 files changed, 17 insertions(+), 20 deletions(-)
--
2.20.1
[Qemu-devel] [PATCH v2 1/4] hw/rdma: Fix broken paths to docs/devel/tracing.txt
The tracing.txt file is under "docs/devel" and not "docs". Reviewed-by: Yuval Shaia Signed-off-by: Kamal Heib --- hw/rdma/trace-events | 2 +- hw/rdma/vmw/trace-events | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/rdma/trace-events b/hw/rdma/trace-events index 0fad56c88206..12868d8a8737 100644 --- a/hw/rdma/trace-events +++ b/hw/rdma/trace-events @@ -1,4 +1,4 @@ -# See docs/tracing.txt for syntax documentation. +# See docs/devel/tracing.txt for syntax documentation. # hw/rdma/rdma_backend.c rdma_check_dev_attr(const char *name, int max_bk, int max_fe) "%s: be=%d, fe=%d" diff --git a/hw/rdma/vmw/trace-events b/hw/rdma/vmw/trace-events index 0122266ad766..e846d54359a3 100644 --- a/hw/rdma/vmw/trace-events +++ b/hw/rdma/vmw/trace-events @@ -1,4 +1,4 @@ -# See docs/tracing.txt for syntax documentation. +# See docs/devel/tracing.txt for syntax documentation. # hw/rdma/vmw/pvrdma_main.c pvrdma_regs_read(uint64_t addr, uint64_t val) "pvrdma.regs[0x%"PRIx64"]=0x%"PRIx64 -- 2.20.1
Re: [Qemu-devel] [PATCH 3/4] hw/rdma: Use {} instead of {0}
On 3/13/19 2:25 PM, Eric Blake wrote:
> On 3/13/19 3:46 AM, Kamal Heib wrote:
>> Signed-off-by: Kamal Heib
>> ---
>> hw/rdma/rdma_backend.c | 18 +-
>> hw/rdma/vmw/pvrdma_cmd.c| 2 +-
>> hw/rdma/vmw/pvrdma_qp_ops.c | 2 +-
>> 3 files changed, 11 insertions(+), 11 deletions(-)
>
> {0} is standard C (but some versions of clang have a bug that it
> mistakenly warns for certain structs); {} is an extension (but one that
> both gcc and clang support). Did you hit the actual clang bug that
> requires this change, or are you only doing it for consistency?
>
I'm doing that for consistency, because some parts of the code already using {}
while other parts aren't, so this patch makes sure that all of them are aligned
to the same convention.
Re: [Qemu-devel] [PATCH 3/4] hw/rdma: Use {} instead of {0}
On 3/13/19 11:33 AM, Yuval Shaia wrote:
> On Wed, Mar 13, 2019 at 10:46:47AM +0200, Kamal Heib wrote:
>> Signed-off-by: Kamal Heib
>
> To be consist please copy git message as well as header message from
> commit 555b3d67bc ("hw/rdma: modify struct initialization").
>
OK, I'll fix that in v2.
>> ---
>> hw/rdma/rdma_backend.c | 18 +-
>> hw/rdma/vmw/pvrdma_cmd.c| 2 +-
>> hw/rdma/vmw/pvrdma_qp_ops.c | 2 +-
>> 3 files changed, 11 insertions(+), 11 deletions(-)
>>
>> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
>> index 78bafc13642a..55b633e451c1 100644
>> --- a/hw/rdma/rdma_backend.c
>> +++ b/hw/rdma/rdma_backend.c
>> @@ -57,7 +57,7 @@ static void dummy_comp_handler(void *ctx, struct ibv_wc
>> *wc)
>> static inline void complete_work(enum ibv_wc_status status, uint32_t
>> vendor_err,
>> void *ctx)
>> {
>> -struct ibv_wc wc = {0};
>> +struct ibv_wc wc = {};
>>
>> wc.status = status;
>> wc.vendor_err = vendor_err;
>> @@ -273,7 +273,7 @@ static void stop_backend_thread(RdmaBackendThread
>> *thread)
>>
>> static void start_comp_thread(RdmaBackendDev *backend_dev)
>> {
>> -char thread_name[THR_NAME_LEN] = {0};
>> +char thread_name[THR_NAME_LEN] = {};
>>
>> stop_backend_thread(_dev->comp_thread);
>>
>> @@ -483,7 +483,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
>> struct ibv_sge new_sge[MAX_SGE];
>> uint32_t bctx_id;
>> int rc;
>> -struct ibv_send_wr wr = {0}, *bad_wr;
>> +struct ibv_send_wr wr = {}, *bad_wr;
>>
>> if (!qp->ibqp) { /* This field is not initialized for QP0 and QP1 */
>> if (qp_type == IBV_QPT_SMI) {
>> @@ -600,7 +600,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
>> struct ibv_sge new_sge[MAX_SGE];
>> uint32_t bctx_id;
>> int rc;
>> -struct ibv_recv_wr wr = {0}, *bad_wr;
>> +struct ibv_recv_wr wr = {}, *bad_wr;
>>
>> if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1
>> */
>> if (qp_type == IBV_QPT_SMI) {
>> @@ -737,7 +737,7 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
>> qp_type,
>> uint32_t max_recv_wr, uint32_t max_send_sge,
>> uint32_t max_recv_sge)
>> {
>> -struct ibv_qp_init_attr attr = {0};
>> +struct ibv_qp_init_attr attr = {};
>>
>> qp->ibqp = 0;
>>
>> @@ -782,7 +782,7 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
>> qp_type,
>> int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP
>> *qp,
>> uint8_t qp_type, uint32_t qkey)
>> {
>> -struct ibv_qp_attr attr = {0};
>> +struct ibv_qp_attr attr = {};
>> int rc, attr_mask;
>>
>> attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
>> @@ -821,7 +821,7 @@ int rdma_backend_qp_state_rtr(RdmaBackendDev
>> *backend_dev, RdmaBackendQP *qp,
>>union ibv_gid *dgid, uint32_t dqpn,
>>uint32_t rq_psn, uint32_t qkey, bool use_qkey)
>> {
>> -struct ibv_qp_attr attr = {0};
>> +struct ibv_qp_attr attr = {};
>> union ibv_gid ibv_gid = {
>> .global.interface_id = dgid->global.interface_id,
>> .global.subnet_prefix = dgid->global.subnet_prefix
>> @@ -880,7 +880,7 @@ int rdma_backend_qp_state_rtr(RdmaBackendDev
>> *backend_dev, RdmaBackendQP *qp,
>> int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
>>uint32_t sq_psn, uint32_t qkey, bool use_qkey)
>> {
>> -struct ibv_qp_attr attr = {0};
>> +struct ibv_qp_attr attr = {};
>> int rc, attr_mask;
>>
>> attr.qp_state = IBV_QPS_RTS;
>> @@ -1012,7 +1012,7 @@ static void process_incoming_mad_req(RdmaBackendDev
>> *backend_dev,
>> complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF,
>>bctx->up_ctx);
>> } else {
>> -struct ibv_wc wc = {0};
>> +struct ibv_wc wc = {};
>> memset(mad, 0, bctx->sge.length);
>> build_mad_hdr((struct ibv_grh *)mad,
>>(union ibv_gid *)>umad.hdr.addr.gid,
>> >hdr.sgid,
>> diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdm
[Qemu-devel] [PATCH 1/4] hw/rdma: Fix broken paths to docs/devel/tracing.txt
The tracing.txt file is under "docs/devel" and not "docs". Signed-off-by: Kamal Heib --- hw/rdma/trace-events | 2 +- hw/rdma/vmw/trace-events | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/rdma/trace-events b/hw/rdma/trace-events index 0fad56c88206..12868d8a8737 100644 --- a/hw/rdma/trace-events +++ b/hw/rdma/trace-events @@ -1,4 +1,4 @@ -# See docs/tracing.txt for syntax documentation. +# See docs/devel/tracing.txt for syntax documentation. # hw/rdma/rdma_backend.c rdma_check_dev_attr(const char *name, int max_bk, int max_fe) "%s: be=%d, fe=%d" diff --git a/hw/rdma/vmw/trace-events b/hw/rdma/vmw/trace-events index 0122266ad766..e846d54359a3 100644 --- a/hw/rdma/vmw/trace-events +++ b/hw/rdma/vmw/trace-events @@ -1,4 +1,4 @@ -# See docs/tracing.txt for syntax documentation. +# See docs/devel/tracing.txt for syntax documentation. # hw/rdma/vmw/pvrdma_main.c pvrdma_regs_read(uint64_t addr, uint64_t val) "pvrdma.regs[0x%"PRIx64"]=0x%"PRIx64 -- 2.20.1
[Qemu-devel] [PATCH 2/4] hw/rdma: Remove unused parameter from rdma_poll_cq()
The 'rdma_dev_res' parameter is not used in rdma_poll_cq(), so remove it.
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index 66185bd487b6..78bafc13642a 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -94,8 +94,7 @@ static void clean_recv_mads(RdmaBackendDev *backend_dev)
} while (cqe_ctx_id != -ENOENT);
}
-static int rdma_poll_cq(RdmaBackendDev *backend_dev,
-RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
+static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
{
int i, ne, total_ne = 0;
BackendCtx *bctx;
@@ -181,7 +180,7 @@ static void *comp_handler_thread(void *arg)
}
backend_dev->rdma_dev_res->stats.poll_cq_from_bk++;
-rdma_poll_cq(backend_dev, backend_dev->rdma_dev_res, ev_cq);
+rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq);
ibv_ack_cq_events(ev_cq, 1);
}
@@ -315,7 +314,7 @@ void rdma_backend_poll_cq(RdmaDeviceResources
*rdma_dev_res, RdmaBackendCQ *cq)
int polled;
rdma_dev_res->stats.poll_cq_from_guest++;
-polled = rdma_poll_cq(cq->backend_dev, rdma_dev_res, cq->ibcq);
+polled = rdma_poll_cq(rdma_dev_res, cq->ibcq);
if (!polled) {
rdma_dev_res->stats.poll_cq_from_guest_empty++;
}
--
2.20.1
[Qemu-devel] [PATCH 3/4] hw/rdma: Use {} instead of {0}
Signed-off-by: Kamal Heib
---
hw/rdma/rdma_backend.c | 18 +-
hw/rdma/vmw/pvrdma_cmd.c| 2 +-
hw/rdma/vmw/pvrdma_qp_ops.c | 2 +-
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
index 78bafc13642a..55b633e451c1 100644
--- a/hw/rdma/rdma_backend.c
+++ b/hw/rdma/rdma_backend.c
@@ -57,7 +57,7 @@ static void dummy_comp_handler(void *ctx, struct ibv_wc *wc)
static inline void complete_work(enum ibv_wc_status status, uint32_t
vendor_err,
void *ctx)
{
-struct ibv_wc wc = {0};
+struct ibv_wc wc = {};
wc.status = status;
wc.vendor_err = vendor_err;
@@ -273,7 +273,7 @@ static void stop_backend_thread(RdmaBackendThread *thread)
static void start_comp_thread(RdmaBackendDev *backend_dev)
{
-char thread_name[THR_NAME_LEN] = {0};
+char thread_name[THR_NAME_LEN] = {};
stop_backend_thread(_dev->comp_thread);
@@ -483,7 +483,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
struct ibv_sge new_sge[MAX_SGE];
uint32_t bctx_id;
int rc;
-struct ibv_send_wr wr = {0}, *bad_wr;
+struct ibv_send_wr wr = {}, *bad_wr;
if (!qp->ibqp) { /* This field is not initialized for QP0 and QP1 */
if (qp_type == IBV_QPT_SMI) {
@@ -600,7 +600,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
struct ibv_sge new_sge[MAX_SGE];
uint32_t bctx_id;
int rc;
-struct ibv_recv_wr wr = {0}, *bad_wr;
+struct ibv_recv_wr wr = {}, *bad_wr;
if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1 */
if (qp_type == IBV_QPT_SMI) {
@@ -737,7 +737,7 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
uint32_t max_recv_wr, uint32_t max_send_sge,
uint32_t max_recv_sge)
{
-struct ibv_qp_init_attr attr = {0};
+struct ibv_qp_init_attr attr = {};
qp->ibqp = 0;
@@ -782,7 +782,7 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t
qp_type,
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey)
{
-struct ibv_qp_attr attr = {0};
+struct ibv_qp_attr attr = {};
int rc, attr_mask;
attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
@@ -821,7 +821,7 @@ int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp,
union ibv_gid *dgid, uint32_t dqpn,
uint32_t rq_psn, uint32_t qkey, bool use_qkey)
{
-struct ibv_qp_attr attr = {0};
+struct ibv_qp_attr attr = {};
union ibv_gid ibv_gid = {
.global.interface_id = dgid->global.interface_id,
.global.subnet_prefix = dgid->global.subnet_prefix
@@ -880,7 +880,7 @@ int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp,
int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
uint32_t sq_psn, uint32_t qkey, bool use_qkey)
{
-struct ibv_qp_attr attr = {0};
+struct ibv_qp_attr attr = {};
int rc, attr_mask;
attr.qp_state = IBV_QPS_RTS;
@@ -1012,7 +1012,7 @@ static void process_incoming_mad_req(RdmaBackendDev
*backend_dev,
complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF,
bctx->up_ctx);
} else {
-struct ibv_wc wc = {0};
+struct ibv_wc wc = {};
memset(mad, 0, bctx->sge.length);
build_mad_hdr((struct ibv_grh *)mad,
(union ibv_gid *)>umad.hdr.addr.gid, >hdr.sgid,
diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
index 21a55e225a61..a0f2b13a2438 100644
--- a/hw/rdma/vmw/pvrdma_cmd.c
+++ b/hw/rdma/vmw/pvrdma_cmd.c
@@ -123,7 +123,7 @@ static int query_port(PVRDMADev *dev, union pvrdma_cmd_req
*req,
{
struct pvrdma_cmd_query_port *cmd = >query_port;
struct pvrdma_cmd_query_port_resp *resp = >query_port_resp;
-struct pvrdma_port_attr attrs = {0};
+struct pvrdma_port_attr attrs = {};
if (cmd->port_num > MAX_PORTS) {
return -EINVAL;
diff --git a/hw/rdma/vmw/pvrdma_qp_ops.c b/hw/rdma/vmw/pvrdma_qp_ops.c
index 508d8fca3c9b..5b9786efbe4b 100644
--- a/hw/rdma/vmw/pvrdma_qp_ops.c
+++ b/hw/rdma/vmw/pvrdma_qp_ops.c
@@ -114,7 +114,7 @@ static void pvrdma_qp_ops_comp_handler(void *ctx, struct
ibv_wc *wc)
static void complete_with_error(uint32_t vendor_err, void *ctx)
{
-struct ibv_wc wc = {0};
+struct ibv_wc wc = {};
wc.status = IBV_WC_GENERAL_ERR;
wc.vendor_err = vendor_err;
--
2.20.1
[Qemu-devel] [PATCH 0/4] pvrdma: misc fixes
The following patch set includes few misc fixes for the pvrdma driver.
Note: this patch set depends on the latest patch set from Yuval.
Kamal Heib (4):
hw/rdma: Fix broken paths to docs/devel/tracing.txt
hw/rdma: Remove unused parameter from rdma_poll_cq()
hw/rdma: Use {} instead of {0}
hw/pvrdma: Fix zero-initialization of resp in {query/modify}_qp
hw/rdma/rdma_backend.c | 25 -
hw/rdma/trace-events| 2 +-
hw/rdma/vmw/pvrdma_cmd.c| 6 ++
hw/rdma/vmw/pvrdma_qp_ops.c | 2 +-
hw/rdma/vmw/trace-events| 2 +-
5 files changed, 17 insertions(+), 20 deletions(-)
--
2.20.1
[Qemu-devel] [PATCH 4/4] hw/pvrdma: Fix zero-initialization of resp in {query/modify}_qp
Make sure to zero-initialize only the pvrdma_cmd_query_qp_resp and not the whole pvrdma_cmd_resp for query_qp, in modify_qp the resp isn't used so remove it. Signed-off-by: Kamal Heib --- hw/rdma/vmw/pvrdma_cmd.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c index a0f2b13a2438..d1ca20a3f90d 100644 --- a/hw/rdma/vmw/pvrdma_cmd.c +++ b/hw/rdma/vmw/pvrdma_cmd.c @@ -488,8 +488,6 @@ static int modify_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, struct pvrdma_cmd_modify_qp *cmd = >modify_qp; int rc; -memset(rsp, 0, sizeof(*rsp)); - /* No need to verify sgid_index since it is u8 */ rc = rdma_rm_modify_qp(>rdma_dev_res, >backend_dev, @@ -512,7 +510,7 @@ static int query_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, struct ibv_qp_init_attr init_attr; int rc; -memset(rsp, 0, sizeof(*rsp)); +memset(resp, 0, sizeof(*resp)); rc = rdma_rm_query_qp(>rdma_dev_res, >backend_dev, cmd->qp_handle, (struct ibv_qp_attr *)>attrs, cmd->attr_mask, -- 2.20.1
Re: [Qemu-devel] [PATCH v6 06/11] hw/rdma: Free all MAD receive buffers when device is closed
On 3/12/19 1:56 PM, Yuval Shaia wrote:
> On Tue, Mar 12, 2019 at 12:08:53PM +0200, Kamal Heib wrote:
>>
>>
>> On 3/11/19 12:29 PM, Yuval Shaia wrote:
>>> When device is going down free all saved MAD buffers.
>>>
>>> Signed-off-by: Yuval Shaia
>>> Reviewed-by: Marcel Apfelbaum
>>> ---
>>> hw/rdma/rdma_backend.c| 34 +-
>>> hw/rdma/vmw/pvrdma_main.c | 2 ++
>>> 2 files changed, 35 insertions(+), 1 deletion(-)
>>>
>>> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
>>> index e8af974..d0bbe57 100644
>>> --- a/hw/rdma/rdma_backend.c
>>> +++ b/hw/rdma/rdma_backend.c
>>> @@ -64,6 +64,33 @@ static inline void complete_work(enum ibv_wc_status
>>> status, uint32_t vendor_err,
>>> comp_handler(ctx, );
>>> }
>>>
>>> +static void free_cqe_ctx(gpointer data, gpointer user_data)
>>> +{
>>> +BackendCtx *bctx;
>>> +RdmaDeviceResources *rdma_dev_res = user_data;
>>
>> No need to do casting for the above assignment "(RdmaDeviceResources *)"?
>
> Compiler didn't gave any error so i skipped it.
>
OK, Other than that:
Reviewed-by: Kamal Heib
>>
>>
>>> +unsigned long cqe_ctx_id = GPOINTER_TO_INT(data);
>>> +
>>> +bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
>>> +if (bctx) {
>>> +rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
>>> +}
>>> +g_free(bctx);
>>> +}
>>> +
>>> +static void clean_recv_mads(RdmaBackendDev *backend_dev)
>>> +{
>>> +unsigned long cqe_ctx_id;
>>> +
>>> +do {
>>> +cqe_ctx_id = rdma_protected_qlist_pop_int64(_dev->
>>> +recv_mads_list);
>>> +if (cqe_ctx_id != -ENOENT) {
>>> +free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
>>> + backend_dev->rdma_dev_res);
>>> +}
>>> +} while (cqe_ctx_id != -ENOENT);
>>> +}
>>> +
>>> static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq
>>> *ibcq)
>>> {
>>> int i, ne, total_ne = 0;
>>> @@ -1037,6 +1064,11 @@ static int mad_init(RdmaBackendDev *backend_dev,
>>> CharBackend *mad_chr_be)
>>> return 0;
>>> }
>>>
>>> +static void mad_stop(RdmaBackendDev *backend_dev)
>>> +{
>>> +clean_recv_mads(backend_dev);
>>> +}
>>> +
>>> static void mad_fini(RdmaBackendDev *backend_dev)
>>> {
>>> disable_rdmacm_mux_async(backend_dev);
>>> @@ -1224,12 +1256,12 @@ void rdma_backend_start(RdmaBackendDev *backend_dev)
>>>
>>> void rdma_backend_stop(RdmaBackendDev *backend_dev)
>>> {
>>> +mad_stop(backend_dev);
>>> stop_backend_thread(_dev->comp_thread);
>>> }
>>>
>>> void rdma_backend_fini(RdmaBackendDev *backend_dev)
>>> {
>>> -rdma_backend_stop(backend_dev);
>>> mad_fini(backend_dev);
>>> g_hash_table_destroy(ah_hash);
>>> ibv_destroy_comp_channel(backend_dev->channel);
>>> diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c
>>> index 729a2df..04845f4 100644
>>> --- a/hw/rdma/vmw/pvrdma_main.c
>>> +++ b/hw/rdma/vmw/pvrdma_main.c
>>> @@ -313,6 +313,8 @@ static void pvrdma_fini(PCIDevice *pdev)
>>>
>>> pvrdma_qp_ops_fini();
>>>
>>> +rdma_backend_stop(>backend_dev);
>>> +
>>> rdma_rm_fini(>rdma_dev_res, >backend_dev,
>>> dev->backend_eth_device_name);
>>>
>>>
>>
>
Re: [Qemu-devel] [PATCH v6 01/11] hw/rdma: Switch to generic error reporting way
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> Utilize error_report for all pr_err calls and some pr_dbg that are
> considered as errors.
> For the remaining pr_dbg calls, the important ones were replaced by
> trace points while other deleted.
>
I suggest updating the commit message to include that some of the functions got
renamed to include prefix "rdma/pvrdma" in the function name.
Other than that:
Reviewed-by: Kamal Heib
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/rdma_backend.c| 354
> +-
> hw/rdma/rdma_backend.h| 4 +-
> hw/rdma/rdma_rm.c | 130 +++-
> hw/rdma/rdma_rm.h | 10 +-
> hw/rdma/rdma_utils.c | 15 +-
> hw/rdma/rdma_utils.h | 45 +-
> hw/rdma/trace-events | 32 +++-
> hw/rdma/vmw/pvrdma.h | 2 +-
> hw/rdma/vmw/pvrdma_cmd.c | 113 --
> hw/rdma/vmw/pvrdma_dev_ring.c | 26 +---
> hw/rdma/vmw/pvrdma_main.c | 144 +++--
> hw/rdma/vmw/pvrdma_qp_ops.c | 49 ++
> hw/rdma/vmw/trace-events | 16 +-
> 13 files changed, 363 insertions(+), 577 deletions(-)
>
> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
> index fd571f2..d138591 100644
> --- a/hw/rdma/rdma_backend.c
> +++ b/hw/rdma/rdma_backend.c
> @@ -14,7 +14,6 @@
> */
>
> #include "qemu/osdep.h"
> -#include "qemu/error-report.h"
> #include "sysemu/sysemu.h"
> #include "qapi/error.h"
> #include "qapi/qmp/qlist.h"
> @@ -39,7 +38,6 @@
>
> typedef struct BackendCtx {
> void *up_ctx;
> -bool is_tx_req;
> struct ibv_sge sge; /* Used to save MAD recv buffer */
> } BackendCtx;
>
> @@ -52,7 +50,7 @@ static void (*comp_handler)(void *ctx, struct ibv_wc *wc);
>
> static void dummy_comp_handler(void *ctx, struct ibv_wc *wc)
> {
> -pr_err("No completion handler is registered\n");
> +rdma_error_report("No completion handler is registered");
> }
>
> static inline void complete_work(enum ibv_wc_status status, uint32_t
> vendor_err,
> @@ -66,29 +64,24 @@ static inline void complete_work(enum ibv_wc_status
> status, uint32_t vendor_err,
> comp_handler(ctx, );
> }
>
> -static void poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
> +static void rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq
> *ibcq)
> {
> int i, ne;
> BackendCtx *bctx;
> struct ibv_wc wc[2];
>
> -pr_dbg("Entering poll_cq loop on cq %p\n", ibcq);
> do {
> ne = ibv_poll_cq(ibcq, ARRAY_SIZE(wc), wc);
>
> -pr_dbg("Got %d completion(s) from cq %p\n", ne, ibcq);
> +trace_rdma_poll_cq(ne, ibcq);
>
> for (i = 0; i < ne; i++) {
> -pr_dbg("wr_id=0x%" PRIx64 "\n", wc[i].wr_id);
> -pr_dbg("status=%d\n", wc[i].status);
> -
> bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, wc[i].wr_id);
> if (unlikely(!bctx)) {
> -pr_dbg("Error: Failed to find ctx for req %" PRId64 "\n",
> - wc[i].wr_id);
> +rdma_error_report("No matching ctx for req %"PRId64,
> + wc[i].wr_id);
> continue;
> }
> -pr_dbg("Processing %s CQE\n", bctx->is_tx_req ? "send" : "recv");
>
> comp_handler(bctx->up_ctx, [i]);
>
> @@ -98,7 +91,7 @@ static void poll_cq(RdmaDeviceResources *rdma_dev_res,
> struct ibv_cq *ibcq)
> } while (ne > 0);
>
> if (ne < 0) {
> -pr_dbg("Got error %d from ibv_poll_cq\n", ne);
> +rdma_error_report("ibv_poll_cq fail, rc=%d, errno=%d", ne, errno);
> }
> }
>
> @@ -115,12 +108,10 @@ static void *comp_handler_thread(void *arg)
> flags = fcntl(backend_dev->channel->fd, F_GETFL);
> rc = fcntl(backend_dev->channel->fd, F_SETFL, flags | O_NONBLOCK);
> if (rc < 0) {
> -pr_dbg("Fail to change to non-blocking mode\n");
> +rdma_error_report("Failed to change backend channel FD to
> non-blocking");
> return NULL;
> }
>
> -pr_dbg("Starting\n");
> -
> pfds[0].fd = backend_dev->channel->fd;
> pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR;
>
> @@ -132,27 +123,25 @@ static void *comp_handler_thread(void *arg)
> } while (!rc && backend_dev->
Re: [Qemu-devel] [PATCH v6 07/11] hw/rdma: Free all receive buffers when QP is destroyed
GSList *list;
> +} RdmaProtectedGSList;
> +
> void *rdma_pci_dma_map(PCIDevice *dev, dma_addr_t addr, dma_addr_t plen);
> void rdma_pci_dma_unmap(PCIDevice *dev, void *buffer, dma_addr_t len);
> void rdma_protected_qlist_init(RdmaProtectedQList *list);
> void rdma_protected_qlist_destroy(RdmaProtectedQList *list);
> void rdma_protected_qlist_append_int64(RdmaProtectedQList *list, int64_t
> value);
> int64_t rdma_protected_qlist_pop_int64(RdmaProtectedQList *list);
> +void rdma_protected_gslist_init(RdmaProtectedGSList *list);
> +void rdma_protected_gslist_destroy(RdmaProtectedGSList *list);
> +void rdma_protected_gslist_append_int32(RdmaProtectedGSList *list,
> +int32_t value);
> +void rdma_protected_gslist_remove_int32(RdmaProtectedGSList *list,
> +int32_t value);
>
> static inline void addrconf_addr_eui48(uint8_t *eui, const char *addr)
> {
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 03/11] hw/rdma: Protect against concurrent execution of poll_cq
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> The function rdma_poll_cq is called from two contexts - completion
> handler thread which sense new completion on backend channel and
> explicitly as result of guest issuing poll_cq command.
>
> Add lock to protect against concurrent executions.
>
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/rdma_backend.c | 2 ++
> hw/rdma/rdma_rm.c | 4
> hw/rdma/rdma_rm_defs.h | 1 +
> 3 files changed, 7 insertions(+)
>
> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
> index 37edf42..1897540 100644
> --- a/hw/rdma/rdma_backend.c
> +++ b/hw/rdma/rdma_backend.c
> @@ -70,6 +70,7 @@ static void rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
> struct ibv_cq *ibcq)
> BackendCtx *bctx;
> struct ibv_wc wc[2];
>
> +qemu_mutex_lock(_dev_res->lock);
> do {
> ne = ibv_poll_cq(ibcq, ARRAY_SIZE(wc), wc);
>
> @@ -89,6 +90,7 @@ static void rdma_poll_cq(RdmaDeviceResources *rdma_dev_res,
> struct ibv_cq *ibcq)
> g_free(bctx);
> }
> } while (ne > 0);
> +qemu_mutex_unlock(_dev_res->lock);
>
> if (ne < 0) {
> rdma_error_report("ibv_poll_cq fail, rc=%d, errno=%d", ne, errno);
> diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
> index 66177b4..7ea62a9 100644
> --- a/hw/rdma/rdma_rm.c
> +++ b/hw/rdma/rdma_rm.c
> @@ -617,12 +617,16 @@ int rdma_rm_init(RdmaDeviceResources *dev_res, struct
> ibv_device_attr *dev_attr)
>
> init_ports(dev_res);
>
> +qemu_mutex_init(_res->lock);
> +
> return 0;
> }
>
> void rdma_rm_fini(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
>const char *ifname)
> {
> +qemu_mutex_destroy(_res->lock);
> +
> fini_ports(dev_res, backend_dev, ifname);
>
> res_tbl_free(_res->uc_tbl);
> diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
> index 0ba61d1..f0ee1f3 100644
> --- a/hw/rdma/rdma_rm_defs.h
> +++ b/hw/rdma/rdma_rm_defs.h
> @@ -105,6 +105,7 @@ typedef struct RdmaDeviceResources {
> RdmaRmResTbl cq_tbl;
> RdmaRmResTbl cqe_ctx_tbl;
> GHashTable *qp_hash; /* Keeps mapping between real and emulated */
> +QemuMutex lock;
> } RdmaDeviceResources;
>
> #endif
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 02/11] hw/rdma: Introduce protected qlist
nmap(dev, buffer, len, DMA_DIRECTION_TO_DEVICE, 0);
> }
> }
> +
> +void rdma_protected_qlist_init(RdmaProtectedQList *list)
> +{
> +qemu_mutex_init(>lock);
> +list->list = qlist_new();
> +}
> +
> +void rdma_protected_qlist_destroy(RdmaProtectedQList *list)
> +{
> +if (list->list) {
> +qlist_destroy_obj(QOBJECT(list->list));
> +qemu_mutex_destroy(>lock);
> +list->list = NULL;
> +}
> +}
> +
> +void rdma_protected_qlist_append_int64(RdmaProtectedQList *list, int64_t
> value)
> +{
> +qemu_mutex_lock(>lock);
> +qlist_append_int(list->list, value);
> +qemu_mutex_unlock(>lock);
> +}
> +
> +int64_t rdma_protected_qlist_pop_int64(RdmaProtectedQList *list)
> +{
> +QObject *obj;
> +
> +qemu_mutex_lock(>lock);
> +obj = qlist_pop(list->list);
> +qemu_mutex_unlock(>lock);
> +
> +if (!obj) {
> +return -ENOENT;
> +}
> +
> +return qnum_get_uint(qobject_to(QNum, obj));
> +}
> diff --git a/hw/rdma/rdma_utils.h b/hw/rdma/rdma_utils.h
> index acd1488..a8bf1d4 100644
> --- a/hw/rdma/rdma_utils.h
> +++ b/hw/rdma/rdma_utils.h
> @@ -29,8 +29,17 @@
> #define rdma_info_report(fmt, ...) \
> info_report("%s: " fmt, "rdma", ## __VA_ARGS__)
>
> +typedef struct RdmaProtectedQList {
> +QemuMutex lock;
> +QList *list;
> +} RdmaProtectedQList;
> +
> void *rdma_pci_dma_map(PCIDevice *dev, dma_addr_t addr, dma_addr_t plen);
> void rdma_pci_dma_unmap(PCIDevice *dev, void *buffer, dma_addr_t len);
> +void rdma_protected_qlist_init(RdmaProtectedQList *list);
> +void rdma_protected_qlist_destroy(RdmaProtectedQList *list);
> +void rdma_protected_qlist_append_int64(RdmaProtectedQList *list, int64_t
> value);
> +int64_t rdma_protected_qlist_pop_int64(RdmaProtectedQList *list);
>
> static inline void addrconf_addr_eui48(uint8_t *eui, const char *addr)
> {
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 06/11] hw/rdma: Free all MAD receive buffers when device is closed
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> When device is going down free all saved MAD buffers.
>
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/rdma_backend.c| 34 +-
> hw/rdma/vmw/pvrdma_main.c | 2 ++
> 2 files changed, 35 insertions(+), 1 deletion(-)
>
> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
> index e8af974..d0bbe57 100644
> --- a/hw/rdma/rdma_backend.c
> +++ b/hw/rdma/rdma_backend.c
> @@ -64,6 +64,33 @@ static inline void complete_work(enum ibv_wc_status
> status, uint32_t vendor_err,
> comp_handler(ctx, );
> }
>
> +static void free_cqe_ctx(gpointer data, gpointer user_data)
> +{
> +BackendCtx *bctx;
> +RdmaDeviceResources *rdma_dev_res = user_data;
No need to do casting for the above assignment "(RdmaDeviceResources *)"?
> +unsigned long cqe_ctx_id = GPOINTER_TO_INT(data);
> +
> +bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
> +if (bctx) {
> +rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
> +}
> +g_free(bctx);
> +}
> +
> +static void clean_recv_mads(RdmaBackendDev *backend_dev)
> +{
> +unsigned long cqe_ctx_id;
> +
> +do {
> +cqe_ctx_id = rdma_protected_qlist_pop_int64(_dev->
> +recv_mads_list);
> +if (cqe_ctx_id != -ENOENT) {
> +free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
> + backend_dev->rdma_dev_res);
> +}
> +} while (cqe_ctx_id != -ENOENT);
> +}
> +
> static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq
> *ibcq)
> {
> int i, ne, total_ne = 0;
> @@ -1037,6 +1064,11 @@ static int mad_init(RdmaBackendDev *backend_dev,
> CharBackend *mad_chr_be)
> return 0;
> }
>
> +static void mad_stop(RdmaBackendDev *backend_dev)
> +{
> +clean_recv_mads(backend_dev);
> +}
> +
> static void mad_fini(RdmaBackendDev *backend_dev)
> {
> disable_rdmacm_mux_async(backend_dev);
> @@ -1224,12 +1256,12 @@ void rdma_backend_start(RdmaBackendDev *backend_dev)
>
> void rdma_backend_stop(RdmaBackendDev *backend_dev)
> {
> +mad_stop(backend_dev);
> stop_backend_thread(_dev->comp_thread);
> }
>
> void rdma_backend_fini(RdmaBackendDev *backend_dev)
> {
> -rdma_backend_stop(backend_dev);
> mad_fini(backend_dev);
> g_hash_table_destroy(ah_hash);
> ibv_destroy_comp_channel(backend_dev->channel);
> diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c
> index 729a2df..04845f4 100644
> --- a/hw/rdma/vmw/pvrdma_main.c
> +++ b/hw/rdma/vmw/pvrdma_main.c
> @@ -313,6 +313,8 @@ static void pvrdma_fini(PCIDevice *pdev)
>
> pvrdma_qp_ops_fini();
>
> +rdma_backend_stop(>backend_dev);
> +
> rdma_rm_fini(>rdma_dev_res, >backend_dev,
> dev->backend_eth_device_name);
>
>
Re: [Qemu-devel] [PATCH v6 08/11] hw/pvrdma: Delete unneeded function argument
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> The function's argument rdma_dev_res is not needed as it is stored in
> the backend_dev object at init.
>
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/rdma_backend.c | 13 ++---
> hw/rdma/rdma_backend.h | 1 -
> hw/rdma/vmw/pvrdma_qp_ops.c | 3 +--
> 3 files changed, 7 insertions(+), 10 deletions(-)
>
> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
> index e124d8d..89279e6 100644
> --- a/hw/rdma/rdma_backend.c
> +++ b/hw/rdma/rdma_backend.c
> @@ -594,7 +594,6 @@ static unsigned int save_mad_recv_buffer(RdmaBackendDev
> *backend_dev,
> }
>
> void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> -RdmaDeviceResources *rdma_dev_res,
> RdmaBackendQP *qp, uint8_t qp_type,
> struct ibv_sge *sge, uint32_t num_sge, void *ctx)
> {
> @@ -613,9 +612,9 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> rc = save_mad_recv_buffer(backend_dev, sge, num_sge, ctx);
> if (rc) {
> complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
> -rdma_dev_res->stats.mad_rx_bufs_err++;
> +backend_dev->rdma_dev_res->stats.mad_rx_bufs_err++;
> } else {
> -rdma_dev_res->stats.mad_rx_bufs++;
> +backend_dev->rdma_dev_res->stats.mad_rx_bufs++;
> }
> }
> return;
> @@ -625,7 +624,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> bctx->up_ctx = ctx;
> bctx->backend_qp = qp;
>
> -rc = rdma_rm_alloc_cqe_ctx(rdma_dev_res, _id, bctx);
> +rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, _id, bctx);
> if (unlikely(rc)) {
> complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
> goto err_free_bctx;
> @@ -633,7 +632,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
>
> rdma_protected_gslist_append_int32(>cqe_ctx_list, bctx_id);
>
> -rc = build_host_sge_array(rdma_dev_res, new_sge, sge, num_sge,
> +rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge,
> num_sge,
>_dev->rdma_dev_res->stats.rx_bufs_len);
> if (rc) {
> complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
> @@ -652,13 +651,13 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> }
>
> atomic_inc(_dev->rdma_dev_res->stats.missing_cqe);
> -rdma_dev_res->stats.rx_bufs++;
> +backend_dev->rdma_dev_res->stats.rx_bufs++;
>
> return;
>
> err_dealloc_cqe_ctx:
> backend_dev->rdma_dev_res->stats.rx_bufs_err++;
> -rdma_rm_dealloc_cqe_ctx(rdma_dev_res, bctx_id);
> +rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
>
> err_free_bctx:
> g_free(bctx);
> diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h
> index c54eaf2..38056d9 100644
> --- a/hw/rdma/rdma_backend.h
> +++ b/hw/rdma/rdma_backend.h
> @@ -111,7 +111,6 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev,
> union ibv_gid *dgid, uint32_t dqpn, uint32_t
> dqkey,
> void *ctx);
> void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> -RdmaDeviceResources *rdma_dev_res,
> RdmaBackendQP *qp, uint8_t qp_type,
> struct ibv_sge *sge, uint32_t num_sge, void
> *ctx);
>
> diff --git a/hw/rdma/vmw/pvrdma_qp_ops.c b/hw/rdma/vmw/pvrdma_qp_ops.c
> index 16db726..508d8fc 100644
> --- a/hw/rdma/vmw/pvrdma_qp_ops.c
> +++ b/hw/rdma/vmw/pvrdma_qp_ops.c
> @@ -231,8 +231,7 @@ void pvrdma_qp_recv(PVRDMADev *dev, uint32_t qp_handle)
> continue;
> }
>
> - rdma_backend_post_recv(>backend_dev, >rdma_dev_res,
> - >backend_qp, qp->qp_type,
> +rdma_backend_post_recv(>backend_dev, >backend_qp,
> qp->qp_type,
> (struct ibv_sge *)>sge[0],
> wqe->hdr.num_sge,
> comp_ctx);
>
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 10/11] hw/pvrdma: Unregister from shutdown notifier when device goes down
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> This hook was installed to close the device when VM is going down.
> After the device is closed there is no need to be informed on VM
> shutdown.
>
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/vmw/pvrdma_main.c | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c
> index a4afced..49bfbd6 100644
> --- a/hw/rdma/vmw/pvrdma_main.c
> +++ b/hw/rdma/vmw/pvrdma_main.c
> @@ -311,6 +311,8 @@ static void pvrdma_fini(PCIDevice *pdev)
> {
> PVRDMADev *dev = PVRDMA_DEV(pdev);
>
> +notifier_remove(>shutdown_notifier);
> +
> pvrdma_qp_ops_fini();
>
> rdma_backend_stop(>backend_dev);
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 11/11] hw/pvrdma: Provide correct value to object_get_typename
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> Use base object of PCIDevice in call to object_get_typename().
>
> Signed-off-by: Yuval Shaia
> ---
> hw/rdma/vmw/pvrdma_main.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c
> index 49bfbd6..0b46561 100644
> --- a/hw/rdma/vmw/pvrdma_main.c
> +++ b/hw/rdma/vmw/pvrdma_main.c
> @@ -593,7 +593,7 @@ static void pvrdma_realize(PCIDevice *pdev, Error **errp)
>
> func0 = pci_get_function_0(pdev);
> /* Break if not vmxnet3 device in slot 0 */
> -if (strcmp(object_get_typename(>qdev.parent_obj), TYPE_VMXNET3)) {
> +if (strcmp(object_get_typename(OBJECT(func0)), TYPE_VMXNET3)) {
> error_setg(errp, "Device on %x.0 must be %s", PCI_SLOT(pdev->devfn),
> TYPE_VMXNET3);
> return;
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 09/11] hw/pvrdma: Delete pvrdma_exit function
On 3/11/19 12:29 PM, Yuval Shaia wrote:
> This hook is not called and was implemented by mistake.
>
> Signed-off-by: Yuval Shaia
> Reviewed-by: Marcel Apfelbaum
> ---
> hw/rdma/vmw/pvrdma_main.c | 6 --
> 1 file changed, 6 deletions(-)
>
> diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c
> index 04845f4..a4afced 100644
> --- a/hw/rdma/vmw/pvrdma_main.c
> +++ b/hw/rdma/vmw/pvrdma_main.c
> @@ -654,11 +654,6 @@ out:
> }
> }
>
> -static void pvrdma_exit(PCIDevice *pdev)
> -{
> -pvrdma_fini(pdev);
> -}
> -
> static void pvrdma_class_init(ObjectClass *klass, void *data)
> {
> DeviceClass *dc = DEVICE_CLASS(klass);
> @@ -666,7 +661,6 @@ static void pvrdma_class_init(ObjectClass *klass, void
> *data)
> RdmaProviderClass *ir = INTERFACE_RDMA_PROVIDER_CLASS(klass);
>
> k->realize = pvrdma_realize;
> -k->exit = pvrdma_exit;
> k->vendor_id = PCI_VENDOR_ID_VMWARE;
> k->device_id = PCI_DEVICE_ID_VMWARE_PVRDMA;
> k->revision = 0x00;
>
Reviewed-by: Kamal Heib
Re: [Qemu-devel] [PATCH v6 04/11] hw/pvrdma: Collect debugging statistics
6 +37,7 @@ static inline void res_tbl_init(const char *name,
> RdmaRmResTbl *tbl,
> tbl->bitmap = bitmap_new(tbl_sz);
> tbl->tbl_sz = tbl_sz;
> tbl->res_sz = res_sz;
> +tbl->used = 0;
> qemu_mutex_init(>lock);
> }
>
> @@ -76,6 +77,8 @@ static inline void *rdma_res_tbl_alloc(RdmaRmResTbl *tbl,
> uint32_t *handle)
>
> set_bit(*handle, tbl->bitmap);
>
> +tbl->used++;
> +
> qemu_mutex_unlock(>lock);
>
> memset(tbl->tbl + *handle * tbl->res_sz, 0, tbl->res_sz);
> @@ -93,6 +96,7 @@ static inline void rdma_res_tbl_dealloc(RdmaRmResTbl *tbl,
> uint32_t handle)
>
> if (handle < tbl->tbl_sz) {
> clear_bit(handle, tbl->bitmap);
> +tbl->used--;
> }
>
> qemu_mutex_unlock(>lock);
> @@ -619,6 +623,9 @@ int rdma_rm_init(RdmaDeviceResources *dev_res, struct
> ibv_device_attr *dev_attr)
>
> qemu_mutex_init(_res->lock);
>
> +memset(_res->stats, 0, sizeof(dev_res->stats));
> +atomic_set(_res->stats.missing_cqe, 0);
> +
> return 0;
> }
>
> diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
> index f0ee1f3..4b8d704 100644
> --- a/hw/rdma/rdma_rm_defs.h
> +++ b/hw/rdma/rdma_rm_defs.h
> @@ -34,7 +34,9 @@
> #define MAX_QP_INIT_RD_ATOM 16
> #define MAX_AH64
>
> -#define MAX_RM_TBL_NAME 16
> +#define MAX_RM_TBL_NAME 16
> +#define MAX_CONSEQ_EMPTY_POLL_CQ4096 /* considered as error above this */
> +
> typedef struct RdmaRmResTbl {
> char name[MAX_RM_TBL_NAME];
> QemuMutex lock;
> @@ -42,6 +44,7 @@ typedef struct RdmaRmResTbl {
> size_t tbl_sz;
> size_t res_sz;
> void *tbl;
> +uint32_t used; /* number of used entries in the table */
> } RdmaRmResTbl;
>
> typedef struct RdmaRmPD {
> @@ -96,6 +99,27 @@ typedef struct RdmaRmPort {
> enum ibv_port_state state;
> } RdmaRmPort;
>
> +typedef struct RdmaRmStats {
> +uint64_t tx;
> +uint64_t tx_len;
> +uint64_t tx_err;
> +uint64_t rx_bufs;
> +uint64_t rx_bufs_len;
> +uint64_t rx_bufs_err;
> +uint64_t completions;
> +uint64_t mad_tx;
> +uint64_t mad_tx_err;
> +uint64_t mad_rx;
> +uint64_t mad_rx_err;
> +uint64_t mad_rx_bufs;
> +uint64_t mad_rx_bufs_err;
> +uint64_t poll_cq_from_bk;
> +uint64_t poll_cq_from_guest;
> +uint64_t poll_cq_from_guest_empty;
> +uint64_t poll_cq_ppoll_to;
> +uint32_t missing_cqe;
> +} RdmaRmStats;
> +
> typedef struct RdmaDeviceResources {
> RdmaRmPort port;
> RdmaRmResTbl pd_tbl;
> @@ -106,6 +130,7 @@ typedef struct RdmaDeviceResources {
> RdmaRmResTbl cqe_ctx_tbl;
> GHashTable *qp_hash; /* Keeps mapping between real and emulated */
> QemuMutex lock;
> +RdmaRmStats stats;
> } RdmaDeviceResources;
>
> #endif
> diff --git a/hw/rdma/vmw/pvrdma.h b/hw/rdma/vmw/pvrdma.h
> index 0879224..a8a04a2 100644
> --- a/hw/rdma/vmw/pvrdma.h
> +++ b/hw/rdma/vmw/pvrdma.h
> @@ -70,6 +70,14 @@ typedef struct DSRInfo {
> PvrdmaRing cq;
> } DSRInfo;
>
> +typedef struct PVRDMADevStats {
> +uint64_t commands;
> +uint64_t regs_reads;
> +uint64_t regs_writes;
> +uint64_t uar_writes;
> +uint64_t interrupts;
> +} PVRDMADevStats;
> +
> typedef struct PVRDMADev {
> PCIDevice parent_obj;
> MemoryRegion msix;
> @@ -89,6 +97,7 @@ typedef struct PVRDMADev {
> CharBackend mad_chr;
> VMXNET3State *func0;
> Notifier shutdown_notifier;
> +PVRDMADevStats stats;
> } PVRDMADev;
> #define PVRDMA_DEV(dev) OBJECT_CHECK(PVRDMADev, (dev), PVRDMA_HW_NAME)
>
> @@ -123,6 +132,7 @@ static inline void post_interrupt(PVRDMADev *dev,
> unsigned vector)
> PCIDevice *pci_dev = PCI_DEVICE(dev);
>
> if (likely(!dev->interrupt_mask)) {
> +dev->stats.interrupts++;
> msix_notify(pci_dev, vector);
> }
> }
> diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
> index 21a55e2..6d56746 100644
> --- a/hw/rdma/vmw/pvrdma_cmd.c
> +++ b/hw/rdma/vmw/pvrdma_cmd.c
> @@ -651,6 +651,8 @@ int pvrdma_exec_cmd(PVRDMADev *dev)
>
> trace_pvrdma_exec_cmd(dsr_info->req->hdr.cmd, dsr_info->rsp->hdr.err);
>
> +dev->stats.commands++;
> +
> out:
> set_reg_val(dev, PVRDMA_REG_ERR, err);
> post_interrupt(dev, INTR_VEC_CMD_RING);
> diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c
> index 81ae08b..dd35646 100644
> --- a/hw/rdma/vmw/pvrdma_main.c
> +++ b/hw/rdma/vmw/pvrdma_main.c
> @@ -337,6 +337,8 @@ static uint64_t pvrdma_regs_read(void *opaque, hwaddr
> addr, unsigned size)
> PVRDMADev *dev = opaque;
> uint32_t val;
>
> +dev->stats.regs_reads++;
> +
> if (get_reg_val(dev, addr, )) {
> rdma_error_report("Failed to read REG value from address 0x%x",
>(uint32_t)addr);
> @@ -353,6 +355,8 @@ static void pvrdma_regs_write(void *opaque, hwaddr addr,
> uint64_t val,
> {
> PVRDMADev *dev = opaque;
>
> +dev->stats.regs_writes++;
> +
> if (set_reg_val(dev, addr, val)) {
> rdma_error_report("Failed to set REG value, addr=0x%"PRIx64 ",
> val=0x%"PRIx64,
>addr, val);
> @@ -421,6 +425,8 @@ static void pvrdma_uar_write(void *opaque, hwaddr addr,
> uint64_t val,
> {
> PVRDMADev *dev = opaque;
>
> +dev->stats.uar_writes++;
> +
> switch (addr & 0xFFF) { /* Mask with 0xFFF as each UC gets page */
> case PVRDMA_UAR_QP_OFFSET:
> if (val & PVRDMA_UAR_QP_SEND) {
> @@ -612,6 +618,8 @@ static void pvrdma_realize(PCIDevice *pdev, Error **errp)
> goto out;
> }
>
> +memset(>stats, 0, sizeof(dev->stats));
> +
> dev->shutdown_notifier.notify = pvrdma_shutdown_notifier;
> qemu_register_shutdown_notifier(>shutdown_notifier);
>
>
Reviewed-by: Kamal Heib
[Qemu-devel] [PATCH] hw/rdma: Fix the error prints in create_qp_rings()
The prints should indicate that we are talking about QP and not CQ.
Fixes: 98d176f8e592 ("hw/rdma: PVRDMA commands and data-path ops")
Reviewed-by: Yuval Shaia
Signed-off-by: Kamal Heib
---
hw/rdma/vmw/pvrdma_cmd.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/hw/rdma/vmw/pvrdma_cmd.c b/hw/rdma/vmw/pvrdma_cmd.c
index 21a55e225a61..8bc24e952073 100644
--- a/hw/rdma/vmw/pvrdma_cmd.c
+++ b/hw/rdma/vmw/pvrdma_cmd.c
@@ -374,13 +374,13 @@ static int create_qp_rings(PCIDevice *pci_dev, uint64_t
pdir_dma,
dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE);
if (!dir) {
-rdma_error_report("Failed to map to CQ page directory");
+rdma_error_report("Failed to map to QP page directory");
goto out;
}
tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE);
if (!tbl) {
-rdma_error_report("Failed to map to CQ page table");
+rdma_error_report("Failed to map to QP page table");
goto out;
}
@@ -393,7 +393,7 @@ static int create_qp_rings(PCIDevice *pci_dev, uint64_t
pdir_dma,
sr->ring_state = (struct pvrdma_ring *)
rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE);
if (!sr->ring_state) {
-rdma_error_report("Failed to map to CQ ring state");
+rdma_error_report("Failed to map to QP ring state");
goto out_free_sr_mem;
}
--
2.20.1
Re: [Qemu-devel] [PATCH] hw/rdma: another clang compilation fix
On 2/14/19 5:40 PM, Marcel Apfelbaum wrote:
> Configuring QEMU with:
>configure --target-list="x86_64-softmmu" --cc=clang --enable-pvrdma
> Results in:
>qemu/hw/rdma/rdma_rm_defs.h:108:3: error: redefinition of typedef
> 'RdmaDeviceResources' is a C11 feature [-Werror,-Wtypedef-redefinition]
>} RdmaDeviceResources;
> ^
>qemu/hw/rdma/rdma_backend_defs.h:24:36: note: previous definition is here
>typedef struct RdmaDeviceResources RdmaDeviceResources;
>
> Fix by removing one of the 'typedef' definitions.
>
> Signed-off-by: Marcel Apfelbaum
> ---
> hw/rdma/rdma_rm_defs.h | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
> index 0ba61d1838..5e877c1fd9 100644
> --- a/hw/rdma/rdma_rm_defs.h
> +++ b/hw/rdma/rdma_rm_defs.h
> @@ -96,7 +96,7 @@ typedef struct RdmaRmPort {
> enum ibv_port_state state;
> } RdmaRmPort;
>
> -typedef struct RdmaDeviceResources {
> +struct RdmaDeviceResources {
> RdmaRmPort port;
> RdmaRmResTbl pd_tbl;
> RdmaRmResTbl mr_tbl;
> @@ -105,6 +105,6 @@ typedef struct RdmaDeviceResources {
> RdmaRmResTbl cq_tbl;
> RdmaRmResTbl cqe_ctx_tbl;
> GHashTable *qp_hash; /* Keeps mapping between real and emulated */
> -} RdmaDeviceResources;
> +};
>
> #endif
>
Acked-by: Kamal Heib
[Qemu-devel] [PATCH] docs/pvrdma: Update rdmacm-mux documentation
Before running the rdmacm-mux need to make sure that both the ib_cm and rdma_cm kernel modules are unloaded. Reviewed-by: Yuval Shaia Signed-off-by: Kamal Heib --- docs/pvrdma.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pvrdma.txt b/docs/pvrdma.txt index 5175251b47d6..81959378b120 100644 --- a/docs/pvrdma.txt +++ b/docs/pvrdma.txt @@ -99,6 +99,9 @@ MAD layer to send and receive RDMA-CM MAD packets. To build rdmacm-mux run # make rdmacm-mux +Before running the rdmacm-mux make sure that both ib_cm and rdma_cm kernel +modules aren't loaded, otherwise the rdmacm-mux service will fail to start. + The application accepts 3 command line arguments and exposes a UNIX socket to pass control and data to it. -d rdma-device-name Name of RDMA device to register with -- 2.20.1
