Re: [PATCH] libibverbs: Add the use of IBV_SEND_INLINE to example pingpong programs
4th bump... On Jul 10, 2013, at 4:32 PM, Jeff Squyres jsquy...@cisco.com wrote: If the send size is less than the cap.max_inline_data reported by the qp, use the IBV_SEND_INLINE flag. This now only shows the example of using ibv_query_qp(), it also reduces the latency time shown by the pingpong programs when the sends can be inlined. Signed-off-by: Jeff Squyres jsquy...@cisco.com --- examples/rc_pingpong.c | 18 +- examples/srq_pingpong.c | 19 +-- examples/uc_pingpong.c | 17 - examples/ud_pingpong.c | 18 +- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c index 15494a1..a8637a5 100644 --- a/examples/rc_pingpong.c +++ b/examples/rc_pingpong.c @@ -65,6 +65,7 @@ struct pingpong_context { struct ibv_qp *qp; void*buf; int size; + int send_flags; int rx_depth; int pending; struct ibv_port_attr portinfo; @@ -319,8 +320,9 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -367,7 +369,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .cap = { @@ -379,11 +382,16 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp = ibv_create_qp(ctx-pd, attr); + ctx-qp = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp) { fprintf(stderr, Couldn't create QP\n); goto clean_cq; } + + ibv_query_qp(ctx-qp, attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } { @@ -508,7 +516,7 @@ static int pp_post_send(struct pingpong_context *ctx) .sg_list= list, .num_sge= 1, .opcode = IBV_WR_SEND, - .send_flags = IBV_SEND_SIGNALED, + .send_flags = ctx-send_flags, }; struct ibv_send_wr *bad_wr; diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c index 6e00f8c..552a144 100644 --- a/examples/srq_pingpong.c +++ b/examples/srq_pingpong.c @@ -68,6 +68,7 @@ struct pingpong_context { struct ibv_qp *qp[MAX_QP]; void*buf; int size; + int send_flags; int num_qp; int rx_depth; int pending[MAX_QP]; @@ -350,9 +351,10 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-num_qp = num_qp; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-num_qp = num_qp; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -413,7 +415,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } for (i = 0; i num_qp; ++i) { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .srq = ctx-srq, @@ -424,11 +427,15 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp[i] = ibv_create_qp(ctx-pd, attr); + ctx-qp[i] = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp[i]) { fprintf(stderr, Couldn't create QP[%d]\n, i); goto clean_qps; } + ibv_query_qp(ctx-qp[i], attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } for (i = 0; i num_qp; ++i) { @@ -568,7 +575,7 @@ static int pp_post_send(struct
Re: [PATCH] libibverbs: Add the use of IBV_SEND_INLINE to example pingpong programs
Bump bump bump. I know this isn't a huge / important patch, but it is a small thing that does decrease the latency reported by these example programs. On Jul 10, 2013, at 4:32 PM, Jeff Squyres jsquy...@cisco.com wrote: If the send size is less than the cap.max_inline_data reported by the qp, use the IBV_SEND_INLINE flag. This not only shows the example of using ibv_query_qp(), it also reduces the latency time shown by the pingpong programs when the sends can be inlined. Signed-off-by: Jeff Squyres jsquy...@cisco.com --- examples/rc_pingpong.c | 18 +- examples/srq_pingpong.c | 19 +-- examples/uc_pingpong.c | 17 - examples/ud_pingpong.c | 18 +- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c index 15494a1..a8637a5 100644 --- a/examples/rc_pingpong.c +++ b/examples/rc_pingpong.c @@ -65,6 +65,7 @@ struct pingpong_context { struct ibv_qp *qp; void*buf; int size; + int send_flags; int rx_depth; int pending; struct ibv_port_attr portinfo; @@ -319,8 +320,9 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -367,7 +369,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .cap = { @@ -379,11 +382,16 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp = ibv_create_qp(ctx-pd, attr); + ctx-qp = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp) { fprintf(stderr, Couldn't create QP\n); goto clean_cq; } + + ibv_query_qp(ctx-qp, attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } { @@ -508,7 +516,7 @@ static int pp_post_send(struct pingpong_context *ctx) .sg_list= list, .num_sge= 1, .opcode = IBV_WR_SEND, - .send_flags = IBV_SEND_SIGNALED, + .send_flags = ctx-send_flags, }; struct ibv_send_wr *bad_wr; diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c index 6e00f8c..552a144 100644 --- a/examples/srq_pingpong.c +++ b/examples/srq_pingpong.c @@ -68,6 +68,7 @@ struct pingpong_context { struct ibv_qp *qp[MAX_QP]; void*buf; int size; + int send_flags; int num_qp; int rx_depth; int pending[MAX_QP]; @@ -350,9 +351,10 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-num_qp = num_qp; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-num_qp = num_qp; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -413,7 +415,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } for (i = 0; i num_qp; ++i) { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .srq = ctx-srq, @@ -424,11 +427,15 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp[i] = ibv_create_qp(ctx-pd, attr); + ctx-qp[i] = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp[i]) { fprintf(stderr, Couldn't create QP[%d]\n, i); goto clean_qps; } + ibv_query_qp(ctx-qp[i], attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |=
Re: [PATCH] libibverbs: Add the use of IBV_SEND_INLINE to example pingpong programs
Bump bump. On Jul 10, 2013, at 4:32 PM, Jeff Squyres jsquy...@cisco.com wrote: If the send size is less than the cap.max_inline_data reported by the qp, use the IBV_SEND_INLINE flag. This now only shows the example of using ibv_query_qp(), it also reduces the latency time shown by the pingpong programs when the sends can be inlined. Signed-off-by: Jeff Squyres jsquy...@cisco.com --- examples/rc_pingpong.c | 18 +- examples/srq_pingpong.c | 19 +-- examples/uc_pingpong.c | 17 - examples/ud_pingpong.c | 18 +- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c index 15494a1..a8637a5 100644 --- a/examples/rc_pingpong.c +++ b/examples/rc_pingpong.c @@ -65,6 +65,7 @@ struct pingpong_context { struct ibv_qp *qp; void*buf; int size; + int send_flags; int rx_depth; int pending; struct ibv_port_attr portinfo; @@ -319,8 +320,9 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -367,7 +369,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .cap = { @@ -379,11 +382,16 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp = ibv_create_qp(ctx-pd, attr); + ctx-qp = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp) { fprintf(stderr, Couldn't create QP\n); goto clean_cq; } + + ibv_query_qp(ctx-qp, attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } { @@ -508,7 +516,7 @@ static int pp_post_send(struct pingpong_context *ctx) .sg_list= list, .num_sge= 1, .opcode = IBV_WR_SEND, - .send_flags = IBV_SEND_SIGNALED, + .send_flags = ctx-send_flags, }; struct ibv_send_wr *bad_wr; diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c index 6e00f8c..552a144 100644 --- a/examples/srq_pingpong.c +++ b/examples/srq_pingpong.c @@ -68,6 +68,7 @@ struct pingpong_context { struct ibv_qp *qp[MAX_QP]; void*buf; int size; + int send_flags; int num_qp; int rx_depth; int pending[MAX_QP]; @@ -350,9 +351,10 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-num_qp = num_qp; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-num_qp = num_qp; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -413,7 +415,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } for (i = 0; i num_qp; ++i) { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .srq = ctx-srq, @@ -424,11 +427,15 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp[i] = ibv_create_qp(ctx-pd, attr); + ctx-qp[i] = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp[i]) { fprintf(stderr, Couldn't create QP[%d]\n, i); goto clean_qps; } + ibv_query_qp(ctx-qp[i], attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } for (i = 0; i num_qp; ++i) { @@ -568,7 +575,7 @@ static int pp_post_send(struct
Re: [PATCH] libibverbs: Add the use of IBV_SEND_INLINE to example pingpong programs
Bump. On Jul 10, 2013, at 4:32 PM, Jeff Squyres jsquy...@cisco.com wrote: If the send size is less than the cap.max_inline_data reported by the qp, use the IBV_SEND_INLINE flag. This now only shows the example of using ibv_query_qp(), it also reduces the latency time shown by the pingpong programs when the sends can be inlined. Signed-off-by: Jeff Squyres jsquy...@cisco.com --- examples/rc_pingpong.c | 18 +- examples/srq_pingpong.c | 19 +-- examples/uc_pingpong.c | 17 - examples/ud_pingpong.c | 18 +- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c index 15494a1..a8637a5 100644 --- a/examples/rc_pingpong.c +++ b/examples/rc_pingpong.c @@ -65,6 +65,7 @@ struct pingpong_context { struct ibv_qp *qp; void*buf; int size; + int send_flags; int rx_depth; int pending; struct ibv_port_attr portinfo; @@ -319,8 +320,9 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -367,7 +369,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .cap = { @@ -379,11 +382,16 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp = ibv_create_qp(ctx-pd, attr); + ctx-qp = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp) { fprintf(stderr, Couldn't create QP\n); goto clean_cq; } + + ibv_query_qp(ctx-qp, attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } { @@ -508,7 +516,7 @@ static int pp_post_send(struct pingpong_context *ctx) .sg_list= list, .num_sge= 1, .opcode = IBV_WR_SEND, - .send_flags = IBV_SEND_SIGNALED, + .send_flags = ctx-send_flags, }; struct ibv_send_wr *bad_wr; diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c index 6e00f8c..552a144 100644 --- a/examples/srq_pingpong.c +++ b/examples/srq_pingpong.c @@ -68,6 +68,7 @@ struct pingpong_context { struct ibv_qp *qp[MAX_QP]; void*buf; int size; + int send_flags; int num_qp; int rx_depth; int pending[MAX_QP]; @@ -350,9 +351,10 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-num_qp = num_qp; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-num_qp = num_qp; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -413,7 +415,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } for (i = 0; i num_qp; ++i) { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .srq = ctx-srq, @@ -424,11 +427,15 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp[i] = ibv_create_qp(ctx-pd, attr); + ctx-qp[i] = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp[i]) { fprintf(stderr, Couldn't create QP[%d]\n, i); goto clean_qps; } + ibv_query_qp(ctx-qp[i], attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } for (i = 0; i num_qp; ++i) { @@ -568,7 +575,7 @@ static int pp_post_send(struct
[PATCH] libibverbs: Add the use of IBV_SEND_INLINE to example pingpong programs
If the send size is less than the cap.max_inline_data reported by the qp, use the IBV_SEND_INLINE flag. This now only shows the example of using ibv_query_qp(), it also reduces the latency time shown by the pingpong programs when the sends can be inlined. Signed-off-by: Jeff Squyres jsquy...@cisco.com --- examples/rc_pingpong.c | 18 +- examples/srq_pingpong.c | 19 +-- examples/uc_pingpong.c | 17 - examples/ud_pingpong.c | 18 +- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c index 15494a1..a8637a5 100644 --- a/examples/rc_pingpong.c +++ b/examples/rc_pingpong.c @@ -65,6 +65,7 @@ struct pingpong_context { struct ibv_qp *qp; void*buf; int size; + int send_flags; int rx_depth; int pending; struct ibv_port_attr portinfo; @@ -319,8 +320,9 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -367,7 +369,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .cap = { @@ -379,11 +382,16 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp = ibv_create_qp(ctx-pd, attr); + ctx-qp = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp) { fprintf(stderr, Couldn't create QP\n); goto clean_cq; } + + ibv_query_qp(ctx-qp, attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } { @@ -508,7 +516,7 @@ static int pp_post_send(struct pingpong_context *ctx) .sg_list= list, .num_sge= 1, .opcode = IBV_WR_SEND, - .send_flags = IBV_SEND_SIGNALED, + .send_flags = ctx-send_flags, }; struct ibv_send_wr *bad_wr; diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c index 6e00f8c..552a144 100644 --- a/examples/srq_pingpong.c +++ b/examples/srq_pingpong.c @@ -68,6 +68,7 @@ struct pingpong_context { struct ibv_qp *qp[MAX_QP]; void*buf; int size; + int send_flags; int num_qp; int rx_depth; int pending[MAX_QP]; @@ -350,9 +351,10 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx) return NULL; - ctx-size = size; - ctx-num_qp = num_qp; - ctx-rx_depth = rx_depth; + ctx-size = size; + ctx-send_flags = IBV_SEND_SIGNALED; + ctx-num_qp = num_qp; + ctx-rx_depth = rx_depth; ctx-buf = memalign(page_size, size); if (!ctx-buf) { @@ -413,7 +415,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, } for (i = 0; i num_qp; ++i) { - struct ibv_qp_init_attr attr = { + struct ibv_qp_attr attr; + struct ibv_qp_init_attr init_attr = { .send_cq = ctx-cq, .recv_cq = ctx-cq, .srq = ctx-srq, @@ -424,11 +427,15 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, .qp_type = IBV_QPT_RC }; - ctx-qp[i] = ibv_create_qp(ctx-pd, attr); + ctx-qp[i] = ibv_create_qp(ctx-pd, init_attr); if (!ctx-qp[i]) { fprintf(stderr, Couldn't create QP[%d]\n, i); goto clean_qps; } + ibv_query_qp(ctx-qp[i], attr, IBV_QP_CAP, init_attr); + if (init_attr.cap.max_inline_data = size) { + ctx-send_flags |= IBV_SEND_INLINE; + } } for (i = 0; i num_qp; ++i) { @@ -568,7 +575,7 @@ static int pp_post_send(struct pingpong_context *ctx, int qp_index)