Re: [PATCH] libibverbs: Add the use of IBV_SEND_INLINE to example pingpong programs
Bump bump.
On Jul 10, 2013, at 4:32 PM, Jeff Squyres jsquy...@cisco.com wrote:
If the send size is less than the cap.max_inline_data reported by the
qp, use the IBV_SEND_INLINE flag. This now only shows the example of
using ibv_query_qp(), it also reduces the latency time shown by the
pingpong programs when the sends can be inlined.
Signed-off-by: Jeff Squyres jsquy...@cisco.com
---
examples/rc_pingpong.c | 18 +-
examples/srq_pingpong.c | 19 +--
examples/uc_pingpong.c | 17 -
examples/ud_pingpong.c | 18 +-
4 files changed, 51 insertions(+), 21 deletions(-)
diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c
index 15494a1..a8637a5 100644
--- a/examples/rc_pingpong.c
+++ b/examples/rc_pingpong.c
@@ -65,6 +65,7 @@ struct pingpong_context {
struct ibv_qp *qp;
void*buf;
int size;
+ int send_flags;
int rx_depth;
int pending;
struct ibv_port_attr portinfo;
@@ -319,8 +320,9 @@ static struct pingpong_context *pp_init_ctx(struct
ibv_device *ib_dev, int size,
if (!ctx)
return NULL;
- ctx-size = size;
- ctx-rx_depth = rx_depth;
+ ctx-size = size;
+ ctx-send_flags = IBV_SEND_SIGNALED;
+ ctx-rx_depth = rx_depth;
ctx-buf = memalign(page_size, size);
if (!ctx-buf) {
@@ -367,7 +369,8 @@ static struct pingpong_context *pp_init_ctx(struct
ibv_device *ib_dev, int size,
}
{
- struct ibv_qp_init_attr attr = {
+ struct ibv_qp_attr attr;
+ struct ibv_qp_init_attr init_attr = {
.send_cq = ctx-cq,
.recv_cq = ctx-cq,
.cap = {
@@ -379,11 +382,16 @@ static struct pingpong_context *pp_init_ctx(struct
ibv_device *ib_dev, int size,
.qp_type = IBV_QPT_RC
};
- ctx-qp = ibv_create_qp(ctx-pd, attr);
+ ctx-qp = ibv_create_qp(ctx-pd, init_attr);
if (!ctx-qp) {
fprintf(stderr, Couldn't create QP\n);
goto clean_cq;
}
+
+ ibv_query_qp(ctx-qp, attr, IBV_QP_CAP, init_attr);
+ if (init_attr.cap.max_inline_data = size) {
+ ctx-send_flags |= IBV_SEND_INLINE;
+ }
}
{
@@ -508,7 +516,7 @@ static int pp_post_send(struct pingpong_context *ctx)
.sg_list= list,
.num_sge= 1,
.opcode = IBV_WR_SEND,
- .send_flags = IBV_SEND_SIGNALED,
+ .send_flags = ctx-send_flags,
};
struct ibv_send_wr *bad_wr;
diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c
index 6e00f8c..552a144 100644
--- a/examples/srq_pingpong.c
+++ b/examples/srq_pingpong.c
@@ -68,6 +68,7 @@ struct pingpong_context {
struct ibv_qp *qp[MAX_QP];
void*buf;
int size;
+ int send_flags;
int num_qp;
int rx_depth;
int pending[MAX_QP];
@@ -350,9 +351,10 @@ static struct pingpong_context *pp_init_ctx(struct
ibv_device *ib_dev, int size,
if (!ctx)
return NULL;
- ctx-size = size;
- ctx-num_qp = num_qp;
- ctx-rx_depth = rx_depth;
+ ctx-size = size;
+ ctx-send_flags = IBV_SEND_SIGNALED;
+ ctx-num_qp = num_qp;
+ ctx-rx_depth = rx_depth;
ctx-buf = memalign(page_size, size);
if (!ctx-buf) {
@@ -413,7 +415,8 @@ static struct pingpong_context *pp_init_ctx(struct
ibv_device *ib_dev, int size,
}
for (i = 0; i num_qp; ++i) {
- struct ibv_qp_init_attr attr = {
+ struct ibv_qp_attr attr;
+ struct ibv_qp_init_attr init_attr = {
.send_cq = ctx-cq,
.recv_cq = ctx-cq,
.srq = ctx-srq,
@@ -424,11 +427,15 @@ static struct pingpong_context *pp_init_ctx(struct
ibv_device *ib_dev, int size,
.qp_type = IBV_QPT_RC
};
- ctx-qp[i] = ibv_create_qp(ctx-pd, attr);
+ ctx-qp[i] = ibv_create_qp(ctx-pd, init_attr);
if (!ctx-qp[i]) {
fprintf(stderr, Couldn't create QP[%d]\n, i);
goto clean_qps;
}
+ ibv_query_qp(ctx-qp[i], attr, IBV_QP_CAP, init_attr);
+ if (init_attr.cap.max_inline_data = size) {
+ ctx-send_flags |= IBV_SEND_INLINE;
+ }
}
for (i = 0; i num_qp; ++i) {
@@ -568,7 +575,7 @@ static int pp_post_send(struct
[PATCH v2] IB/qib: add err_decode() call for ring dump
Commit b3ddf380 (Log all SDMA errors unconditionally) missed part of the patch. This also corrects a format warning when dma_addr_t is 32 bits on a 64 bit system. Signed-off-by: Mike Marciniszyn mike.marcinis...@intel.com --- drivers/infiniband/hw/qib/qib_iba7322.c |2 ++ drivers/infiniband/hw/qib/qib_sdma.c|2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index 21e8b09..016e742 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -1596,6 +1596,8 @@ static void sdma_7322_p_errors(struct qib_pportdata *ppd, u64 errs) struct qib_devdata *dd = ppd-dd; errs = QIB_E_P_SDMAERRS; + err_decode(ppd-cpspec-sdmamsgbuf, sizeof(ppd-cpspec-sdmamsgbuf), + errs, qib_7322p_error_msgs); if (errs QIB_E_P_SDMAUNEXPDATA) qib_dev_err(dd, IB%u:%u SDmaUnexpData\n, dd-unit, diff --git a/drivers/infiniband/hw/qib/qib_sdma.c b/drivers/infiniband/hw/qib/qib_sdma.c index f63b2ac..c6d6a54 100644 --- a/drivers/infiniband/hw/qib/qib_sdma.c +++ b/drivers/infiniband/hw/qib/qib_sdma.c @@ -723,7 +723,7 @@ void dump_sdma_state(struct qib_pportdata *ppd) struct qib_sdma_txreq *txp, *txpnext; __le64 *descqp; u64 desc[2]; - dma_addr_t addr; + u64 addr; u16 gen, dwlen, dwoffset; u16 head, tail, cnt; -- To unsubscribe from this list: send the line unsubscribe linux-rdma in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
RE: [PATCH ibacm] Update documentation for the new preload option names
thanks - committed -- To unsubscribe from this list: send the line unsubscribe linux-rdma in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH] IB/qib: This patch improves SDMA performance
From: CQ Tang cq.t...@intel.com
1. The code accepts chunks of messages, and splits the chunk into
packets when converting packets into sdma queue entries, adjacent
packets will use user buffer pages smartly to avoid pinning the
same page multiple times.
2. Instead of discarding all the work when SDMA queue is full, the
work is saved in a pending queue, whenever there is enough SDMA
queue free entries, pending queue is directly put onto SDMA queue.
3. An interrupt handler is used to progress this pending queue.
Reviewed-by: Mike Marciniszyn mike.marcinis...@intel.com
Signed-off-by: CQ Tang cq.t...@intel.com
Signed-off-by: Mike Marciniszyn mike.marcinis...@intel.com
---
drivers/infiniband/hw/qib/qib.h |4
drivers/infiniband/hw/qib/qib_common.h| 32 +
drivers/infiniband/hw/qib/qib_file_ops.c |2
drivers/infiniband/hw/qib/qib_sdma.c |8
drivers/infiniband/hw/qib/qib_user_sdma.c | 909 ++---
5 files changed, 728 insertions(+), 227 deletions(-)
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 4a9af79..ae3e4fe 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -576,11 +576,13 @@ struct qib_pportdata {
/* read/write using lock */
spinlock_tsdma_lock cacheline_aligned_in_smp;
struct list_head sdma_activelist;
+ struct list_head sdma_userpending;
u64 sdma_descq_added;
u64 sdma_descq_removed;
u16 sdma_descq_tail;
u16 sdma_descq_head;
u8sdma_generation;
+ u8sdma_intrequest;
struct tasklet_struct sdma_sw_clean_up_task
cacheline_aligned_in_smp;
@@ -1326,6 +1328,8 @@ int qib_setup_sdma(struct qib_pportdata *);
void qib_teardown_sdma(struct qib_pportdata *);
void __qib_sdma_intr(struct qib_pportdata *);
void qib_sdma_intr(struct qib_pportdata *);
+void qib_user_sdma_send_desc(struct qib_pportdata *dd,
+ struct list_head *pktlist);
int qib_sdma_verbs_send(struct qib_pportdata *, struct qib_sge_state *,
u32, struct qib_verbs_txreq *);
/* ppd-sdma_lock should be locked before calling this. */
diff --git a/drivers/infiniband/hw/qib/qib_common.h
b/drivers/infiniband/hw/qib/qib_common.h
index 4f255b7..5670ace 100644
--- a/drivers/infiniband/hw/qib/qib_common.h
+++ b/drivers/infiniband/hw/qib/qib_common.h
@@ -279,7 +279,7 @@ struct qib_base_info {
* may not be implemented; the user code must deal with this if it
* cares, or it must abort after initialization reports the difference.
*/
-#define QIB_USER_SWMINOR 12
+#define QIB_USER_SWMINOR 13
#define QIB_USER_SWVERSION ((QIB_USER_SWMAJOR 16) | QIB_USER_SWMINOR)
@@ -701,7 +701,37 @@ struct qib_message_header {
__be32 bth[3];
/* fields below this point are in host byte order */
struct qib_header iph;
+ /* fields below are simplified, but should match PSM */
+ /* some are accessed by driver when packet spliting is needed */
__u8 sub_opcode;
+ __u8 flags;
+ __u16 commidx;
+ __u32 ack_seq_num;
+ __u8 flowid;
+ __u8 hdr_dlen;
+ __u16 mqhdr;
+ __u32 uwords[4];
+};
+
+/* sequence number bits for message */
+union qib_seqnum {
+ struct {
+ __u32 seq:11;
+ __u32 gen:8;
+ __u32 flow:5;
+ };
+ struct {
+ __u32 pkt:16;
+ __u32 msg:8;
+ };
+ __u32 val;
+};
+
+/* qib receiving-dma tid-session-member */
+struct qib_tid_session_member {
+ __u16 tid;
+ __u16 offset;
+ __u16 length;
};
/* IB - LRH header consts */
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c
b/drivers/infiniband/hw/qib/qib_file_ops.c
index df3808a..d06e33a 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -1220,7 +1220,7 @@ static int qib_compatible_subctxts(int user_swmajor, int
user_swminor)
return user_swminor == 3;
default:
/* = 4 are compatible (or are expected to be) */
- return user_swminor = 4;
+ return user_swminor = QIB_USER_SWMINOR;
}
}
/* make no promises yet for future major versions */
diff --git a/drivers/infiniband/hw/qib/qib_sdma.c
b/drivers/infiniband/hw/qib/qib_sdma.c
index 32162d3..f63b2ac 100644
--- a/drivers/infiniband/hw/qib/qib_sdma.c
+++ b/drivers/infiniband/hw/qib/qib_sdma.c
@@ -423,8 +423,11 @@ void qib_sdma_intr(struct qib_pportdata *ppd)
void __qib_sdma_intr(struct qib_pportdata *ppd)
{
- if (__qib_sdma_running(ppd))
+ if (__qib_sdma_running(ppd)) {
qib_sdma_make_progress(ppd);
+ if
[PATCH ibacm] Add support for name cache
Similar to IPv4 and IPv6 preload, (host)name cache preload is
now supported in addr_preload_file when acm_hosts is specified
for addr_preload setting.
Signed-off-by: Hal Rosenstock h...@mellanox.com
---
diff --git a/acm_notes.txt b/acm_notes.txt
index d204484..3f24322 100644
--- a/acm_notes.txt
+++ b/acm_notes.txt
@@ -136,7 +136,7 @@ none which does not preload these caches. See
dump_pr.notes.txt in dump_pr
for more information on the opensm_full_v1 file format and how to configure
OpenSM to generate this file.
-Additional IPv4 and IPv6 caches can be be preloaded by using the
-addr_preload option. The default is none which does not preload these
+Additionally, the name, IPv4, and IPv6 caches can be be preloaded by using
+the addr_preload option. The default is none which does not preload these
caches. To preload these caches, set this option to acm_hosts and
configure the addr_data_file appropriately.
diff --git a/ibacm_hosts.data b/ibacm_hosts.data
index 9011691..78b978d 100644
--- a/ibacm_hosts.data
+++ b/ibacm_hosts.data
@@ -3,10 +3,13 @@
# Entry format is:
# address IB GID
#
+# The address may be one of the following:
+# host_name - ascii character string, up to 31 characters
# address - IPv4 or IPv6 formatted address
#
# There can be multiple entries for a single IB GID
#
# Samples:
+# luna3 fe80::8:f104:39a:169
# 192.168.1.3 fe80::8:f104:39a:169
# fe80::208:f104:39a:169 fe80::8:f104:39a:169
diff --git a/man/ibacm.1 b/man/ibacm.1
index 0ff58a2..fb15752 100644
--- a/man/ibacm.1
+++ b/man/ibacm.1
@@ -157,8 +157,8 @@ See dump_pr.notes.txt in dump_pr for more information on the
full_opensm_v1 file format and how to configure OpenSM to
generate this file.
.P
-Additional IPv4 and IPv6 caches can be be preloaded by using the
-addr_preload option. The default is none which does not preload these
+Additionally, the name, IPv4, and IPv6 caches can be be preloaded by using
+the addr_preload option. The default is none which does not preload these
caches. To preload these caches, set this option to acm_hosts and
configure the addr_data_file appropriately.
.SH SEE ALSO
diff --git a/src/acm.c b/src/acm.c
index 7f926b4..09051dc 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -2832,20 +2832,18 @@ static void acm_parse_hosts_file(struct acm_ep *ep)
acm_log(0, ERROR - %s is not IB GID\n, gid);
continue;
}
+ memset(name, 0, ACM_MAX_ADDRESS);
if (inet_pton(AF_INET, addr, ip_addr) 0) {
addr_type = ACM_ADDRESS_IP;
+ memcpy(name, ip_addr, 4);
} else if (inet_pton(AF_INET6, addr, ip_addr) 0) {
addr_type = ACM_ADDRESS_IP6;
+ memcpy(name, ip_addr, sizeof(ip_addr));
} else {
- acm_log(0, ERROR - %s is not IP address\n, addr);
- continue;
+ addr_type = ACM_ADDRESS_NAME;
+ strncpy((char *)name, addr, ACM_MAX_ADDRESS);
}
- memset(name, 0, ACM_MAX_ADDRESS);
- if (addr_type == ACM_ADDRESS_IP)
- memcpy(name, ip_addr, 4);
- else
- memcpy(name, ip_addr, sizeof(ip_addr));
dest = acm_acquire_dest(ep, addr_type, name);
if (!dest) {
acm_log(0, ERROR - unable to create dest %s\n, addr);
--
To unsubscribe from this list: send the line unsubscribe linux-rdma in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH ibacm] Update documentation for the new preload option names
Signed-off-by: Hal Rosenstock h...@mellanox.com --- diff --git a/acm_notes.txt b/acm_notes.txt index 3903b8a..d204484 100644 --- a/acm_notes.txt +++ b/acm_notes.txt @@ -137,5 +137,6 @@ for more information on the opensm_full_v1 file format and how to configure OpenSM to generate this file. Additional IPv4 and IPv6 caches can be be preloaded by using the -addr_preload option. The default is none which does not preload these -caches. +addr_preload option. The default is none which does not preload these +caches. To preload these caches, set this option to acm_hosts and +configure the addr_data_file appropriately. diff --git a/man/ibacm.1 b/man/ibacm.1 index 2ca7eec..0ff58a2 100644 --- a/man/ibacm.1 +++ b/man/ibacm.1 @@ -1,4 +1,4 @@ -.TH ibacm 1 2013-06-18 ibacm ibacm ibacm +.TH ibacm 1 2013-07-18 ibacm ibacm ibacm .SH NAME ibacm \- address and route resolution services for InfiniBand. .SH SYNOPSIS @@ -151,14 +151,15 @@ and LID destination addresses. These caches can be optionally preloaded. ibacm supports the OpenSM dump_pr plugin full PathRecord format which is used to preload these caches. The file format is specified in the ibacm_opts.cfg file via the -path_rec_fmt parameter which should be set to full_opensm_v1 for this file -format. Default format is none which does not preload these caches. +route_preload setting which should be set to full_opensm_v1 for this +file format. Default format is none which does not preload these caches. See dump_pr.notes.txt in dump_pr for more information on the full_opensm_v1 file format and how to configure OpenSM to generate this file. .P Additional IPv4 and IPv6 caches can be be preloaded by using the -hosts_file option. The default is none which does not preload these -caches. +addr_preload option. The default is none which does not preload these +caches. To preload these caches, set this option to acm_hosts and +configure the addr_data_file appropriately. .SH SEE ALSO ibacm(7), ib_acme(1), rdma_cm(7) -- To unsubscribe from this list: send the line unsubscribe linux-rdma in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
