[PATCH 0/7] IBoE user space support
Roland, the enclosed patch series enhances the user space IB stack to support IBoE. This work is based on earlier patches done by Eli Cohen and posted during August/2010 see some pointers below. One notable change from the previous post, is keeping the UD related resolution between GID to Ethernet L2 address objects (e.g mac, vlan) within libmlx4. Another patch with notable change is the last, which aligns libmlx4 with the kernel w.r.t supported devices. [PATCH 1/7] libibverbs: Add link layer field port attribute [PATCH 2/7] libibverbs: change kernel API to accept link layer [PATCH 3/7] libibverbs: add GID change event [PATCH 4/7] libibverbs: update examples for IBoE [PATCH 5/7] libmlx4: add IBoE support [PATCH 6/7] libmlx4: add IBoE UD/VLANs support [PATCH 7/7] libmlx4: align the list of ConnectX devices supported with the kernel Or. libibverbs V10 from August 2010 http://marc.info/?l=linux-rdma&m=128283236815520 http://marc.info/?l=linux-rdma&m=128283237715545 http://marc.info/?l=linux-rdma&m=128283238115562 http://marc.info/?l=linux-rdma&m=128283239215578 libmlx4 V10 from August 2010 http://marc.info/?l=linux-rdma&m=128283239515584 http://marc.info/?l=linux-rdma&m=128283240415602 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 1/7] libibverbs: Add link layer field port attribute
The new field has one of three values - IBV_LINK_LAYER_UNSPECIFIED,
IBV_LINK_LAYER_INFINIBAND, IBV_LINK_LAYER_ETHERNET. It can be used
by applications to know the link layer used by the port, which
can be either Infiniband or Ethernet.
The addition of the new field does not change the size of struct
ibv_port_attr due to alignment of the preceding fields. As such
binary compatibility between the library to applications is kept,
since old apps running over new library do not read this field,
and new apps running over old library will determine the link
layer as unspecified and hence take their IB code path.
The solution was suggested by: Roland Dreier
and Jason Gunthorpe
Signed-off-by: Or Gerlitz
Signed-off-by: Eli Cohen
---
include/infiniband/verbs.h | 21 +
man/ibv_query_port.3 |4
2 files changed, 25 insertions(+), 0 deletions(-)
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 0f1cb2e..17df3ff 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -161,6 +161,12 @@ enum ibv_port_state {
IBV_PORT_ACTIVE_DEFER = 5
};
+enum {
+ IBV_LINK_LAYER_UNSPECIFIED,
+ IBV_LINK_LAYER_INFINIBAND,
+ IBV_LINK_LAYER_ETHERNET,
+};
+
struct ibv_port_attr {
enum ibv_port_state state;
enum ibv_mtumax_mtu;
@@ -181,6 +187,8 @@ struct ibv_port_attr {
uint8_t active_width;
uint8_t active_speed;
uint8_t phys_state;
+ uint8_t link_layer;
+ uint8_t pad;
};
enum ibv_event_type {
@@ -693,6 +701,16 @@ struct ibv_context {
void *abi_compat;
};
+static inline int ___ibv_query_port(struct ibv_context *context,
+ uint8_t port_num,
+ struct ibv_port_attr *port_attr)
+{
+ port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+ port_attr->pad = 0;
+
+ return context->ops.query_port(context, port_num, port_attr);
+}
+
/**
* ibv_get_device_list - Get list of IB devices currently available
* @num_devices: optional. if non-NULL, set to the number of devices
@@ -1097,4 +1115,7 @@ END_C_DECLS
# undef __attribute_const
+#define ibv_query_port(context, port_num, port_attr) \
+ ___ibv_query_port(context, port_num, port_attr)
+
#endif /* INFINIBAND_VERBS_H */
diff --git a/man/ibv_query_port.3 b/man/ibv_query_port.3
index 882470d..9bedd90 100644
--- a/man/ibv_query_port.3
+++ b/man/ibv_query_port.3
@@ -44,9 +44,13 @@ uint8_t init_type_reply;/* Type of
initialization performed by S
uint8_t active_width; /* Currently active link width */
uint8_t active_speed; /* Currently active link speed */
uint8_t phys_state; /* Physical port state */
+uint8_t link_layer; /* link layer protocol of the port */
.in -8
};
.sp
+possible values for the link layer field are IBV_LINK_LAYER_INFINIBAND,
+IBV_LINK_LAYER_ETHERNET, or IBV_LINK_LAYER_UNSPECIFIED.
+.sp
.fi
.SH "RETURN VALUE"
.B ibv_query_port()
--
1.5.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 2/7] libibverbs: change kernel API to accept link layer
Modify the code to allow passing the link layer of a port from the
kernel to the library.
The new field does not change the size of struct ibv_query_port_resp
as it replaces a reserved field. As such binary compatibility between
the kernel to the library is kept, since old kernel running below new
library will not touch that field so it will be resolved as unspecified,
and old library running over new kernel will ignore the setting done by
the kernel.
The solution was suggested by: Roland Dreier
and Jason Gunthorpe
Signed-off-by: Or Gerlitz
Signed-off-by: Eli Cohen
---
include/infiniband/kern-abi.h |3 ++-
src/cmd.c |1 +
2 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 0db083a..619ea7e 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -223,7 +223,8 @@ struct ibv_query_port_resp {
__u8 active_width;
__u8 active_speed;
__u8 phys_state;
- __u8 reserved[3];
+ __u8 link_layer;
+ __u8 reserved[2];
};
struct ibv_alloc_pd {
diff --git a/src/cmd.c b/src/cmd.c
index cbd5288..39af833 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -196,6 +196,7 @@ int ibv_cmd_query_port(struct ibv_context *context, uint8_t
port_num,
port_attr->active_width= resp.active_width;
port_attr->active_speed= resp.active_speed;
port_attr->phys_state = resp.phys_state;
+ port_attr->link_layer = resp.link_layer;
return 0;
}
--
1.5.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 3/7] libibverbs: add GID change event
Add IB GID change event which is generated by the kernel
IBoE stack when the HW driver updates the GID table.
Signed-off-by: Or Gerlitz
Signed-off-by: Eli Cohen
---
examples/asyncwatch.c |2 ++
include/infiniband/verbs.h |3 ++-
man/ibv_get_async_event.3 |2 ++
3 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/examples/asyncwatch.c b/examples/asyncwatch.c
index 5510a29..da7ebd4 100644
--- a/examples/asyncwatch.c
+++ b/examples/asyncwatch.c
@@ -57,6 +57,8 @@ static const char *event_name_str(enum ibv_event_type
event_type)
return "IBV_EVENT_SM_CHANGE";
case IBV_EVENT_CLIENT_REREGISTER:
return "IBV_EVENT_CLIENT_REREGISTER";
+ case IBV_EVENT_GID_CHANGE:
+ return "IBV_EVENT_GID_CHANGE";
case IBV_EVENT_CQ_ERR:
case IBV_EVENT_QP_FATAL:
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 17df3ff..0dd79f6 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -209,7 +209,8 @@ enum ibv_event_type {
IBV_EVENT_SRQ_ERR,
IBV_EVENT_SRQ_LIMIT_REACHED,
IBV_EVENT_QP_LAST_WQE_REACHED,
- IBV_EVENT_CLIENT_REREGISTER
+ IBV_EVENT_CLIENT_REREGISTER,
+ IBV_EVENT_GID_CHANGE,
};
struct ibv_async_event {
diff --git a/man/ibv_get_async_event.3 b/man/ibv_get_async_event.3
index acb6257..a76dc0c 100644
--- a/man/ibv_get_async_event.3
+++ b/man/ibv_get_async_event.3
@@ -81,6 +81,8 @@ following events:
.B IBV_EVENT_SM_CHANGE \fR SM was changed on a port
.TP
.B IBV_EVENT_CLIENT_REREGISTER \fR SM sent a CLIENT_REREGISTER request to a
port
+.TP
+.B IBV_EVENT_GID_CHANGE \fR GID table was changed on a port
.PP
.I CA events:
.TP
--
1.5.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 4/7] libibverbs: Update examples for IBoE
Since IBoE requires usage of GRH, update ibv_*_pinpong examples to accept
GIDs. GIDs are given as an index to the local port's table and are exchanged
between the client and the server through the socket connection.
Signed-off-by: Or Gerlitz
Signed-off-by: Eli Cohen
---
examples/devinfo.c | 15 +++
examples/pingpong.c | 32 +++
examples/pingpong.h |4 ++
examples/rc_pingpong.c | 97 ---
examples/srq_pingpong.c | 96 +++---
examples/uc_pingpong.c | 97 ---
examples/ud_pingpong.c | 89 ---
7 files changed, 342 insertions(+), 88 deletions(-)
diff --git a/examples/devinfo.c b/examples/devinfo.c
index 84f95c7..9c472a0 100644
--- a/examples/devinfo.c
+++ b/examples/devinfo.c
@@ -184,6 +184,19 @@ static int print_all_port_gids(struct ibv_context *ctx,
uint8_t port_num, int tb
return rc;
}
+static const char *link_layer_str(uint8_t link_layer)
+{
+ switch (link_layer) {
+ case IBV_LINK_LAYER_UNSPECIFIED:
+ case IBV_LINK_LAYER_INFINIBAND:
+ return "IB";
+ case IBV_LINK_LAYER_ETHERNET:
+ return "Ethernet";
+ default:
+ return "Unknown";
+ }
+}
+
static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
{
struct ibv_context *ctx;
@@ -284,6 +297,8 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t
ib_port)
printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid);
printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid);
printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc);
+ printf("\t\t\tlink_layer:\t\t%s\n",
+ link_layer_str(port_attr.link_layer));
if (verbose) {
printf("\t\t\tmax_msg_sz:\t\t0x%x\n",
port_attr.max_msg_sz);
diff --git a/examples/pingpong.c b/examples/pingpong.c
index b916f59..d06ba84 100644
--- a/examples/pingpong.c
+++ b/examples/pingpong.c
@@ -31,6 +31,10 @@
*/
#include "pingpong.h"
+#include
+#include
+#include
+#include
enum ibv_mtu pp_mtu_to_enum(int mtu)
{
@@ -53,3 +57,31 @@ uint16_t pp_get_local_lid(struct ibv_context *context, int
port)
return attr.lid;
}
+
+int pp_get_port_info(struct ibv_context *context, int port,
+struct ibv_port_attr *attr)
+{
+ return ibv_query_port(context, port, attr);
+}
+
+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
+{
+ char tmp[9];
+ uint32_t v32;
+ int i;
+
+ for (tmp[8] = 0, i = 0; i < 4; ++i) {
+ memcpy(tmp, wgid + i * 8, 8);
+ sscanf(tmp, "%x", &v32);
+ *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32);
+ }
+}
+
+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
+{
+ int i;
+
+ for (i = 0; i < 4; ++i)
+ sprintf(&wgid[i * 8], "%08x",
+ htonl(*(uint32_t *)(gid->raw + i * 4)));
+}
diff --git a/examples/pingpong.h b/examples/pingpong.h
index 71d7c3f..9cdc03e 100644
--- a/examples/pingpong.h
+++ b/examples/pingpong.h
@@ -37,5 +37,9 @@
enum ibv_mtu pp_mtu_to_enum(int mtu);
uint16_t pp_get_local_lid(struct ibv_context *context, int port);
+int pp_get_port_info(struct ibv_context *context, int port,
+struct ibv_port_attr *attr);
+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid);
+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]);
#endif /* IBV_PINGPONG_H */
diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c
index fa969e0..0b7f3e0 100644
--- a/examples/rc_pingpong.c
+++ b/examples/rc_pingpong.c
@@ -67,17 +67,19 @@ struct pingpong_context {
int size;
int rx_depth;
int pending;
+ struct ibv_port_attr portinfo;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
+ union ibv_gid gid;
};
static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
enum ibv_mtu mtu, int sl,
- struct pingpong_dest *dest)
+ struct pingpong_dest *dest, int sgid_idx)
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_RTR,
@@ -94,6 +96,13 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int
port, int my_psn,
.port_num = port
}
};
+
+ if (dest->gid.global.interface_id) {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.grh.dgid = dest->gid;
+ attr.ah_attr.grh.sgid_index = sgid_idx;
+ }
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE
[PATCH 5/7] libmlx4: add IBoE support
Modify libmlx4 to support IBoE, where the only user space piece
to handle is the creation of UD address handles - the L2 Ethernet
attributes have to be resolved from the DGID.
Derived from work by Eli Cohen
Signed-off-by: Or Gerlitz
---
src/mlx4.h |1 +
src/qp.c|1 +
src/verbs.c | 49 +++--
src/wqe.h |3 ++-
4 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/src/mlx4.h b/src/mlx4.h
index 4445998..b277b06 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -241,6 +241,7 @@ struct mlx4_av {
struct mlx4_ah {
struct ibv_ah ibv_ah;
struct mlx4_av av;
+ uint8_t mac[6];
};
static inline unsigned long align(unsigned long val, unsigned long align)
diff --git a/src/qp.c b/src/qp.c
index ec138cd..4d79e38 100644
--- a/src/qp.c
+++ b/src/qp.c
@@ -144,6 +144,7 @@ static void set_datagram_seg(struct mlx4_wqe_datagram_seg
*dseg,
memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
dseg->dqpn = htonl(wr->wr.ud.remote_qpn);
dseg->qkey = htonl(wr->wr.ud.remote_qkey);
+ memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6);
}
static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg)
diff --git a/src/verbs.c b/src/verbs.c
index 1ac1362..6620ac2 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -614,9 +614,45 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp)
return 0;
}
+static int link_local_gid(const union ibv_gid *gid)
+{
+ uint32_t hi = *(uint32_t *)(gid->raw);
+ uint32_t lo = *(uint32_t *)(gid->raw + 4);
+ if (hi == htonl(0xfe80) && lo == 0)
+ return 1;
+
+ return 0;
+}
+
+static uint16_t get_vlan_id(union ibv_gid *gid)
+{
+ uint16_t vid;
+ vid = gid->raw[11] << 8 | gid->raw[12];
+ return vid < 0x1000 ? vid : 0x;
+}
+
+
+static int mlx4_resolve_grh_to_l2(struct mlx4_ah *ah, struct ibv_ah_attr *attr)
+{
+ if (get_vlan_id(&attr->grh.dgid) != 0x)
+ return 1;
+
+ if (link_local_gid(&attr->grh.dgid)) {
+ memcpy(ah->mac, &attr->grh.dgid.raw[8], 3);
+ memcpy(ah->mac + 3, &attr->grh.dgid.raw[13], 3);
+ ah->mac[0] ^= 2;
+ return 0;
+ } else
+ return 1;
+}
+
struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
{
struct mlx4_ah *ah;
+ struct ibv_port_attr port_attr;
+
+ if (ibv_query_port(pd->context, attr->port_num, &port_attr))
+ return NULL;
ah = malloc(sizeof *ah);
if (!ah)
@@ -625,8 +661,11 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct
ibv_ah_attr *attr)
memset(&ah->av, 0, sizeof ah->av);
ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24));
- ah->av.g_slid= attr->src_path_bits;
- ah->av.dlid = htons(attr->dlid);
+
+ if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
+ ah->av.g_slid = attr->src_path_bits;
+ ah->av.dlid = htons(attr->dlid);
+ }
if (attr->static_rate) {
ah->av.stat_rate = attr->static_rate + MLX4_STAT_RATE_OFFSET;
/* XXX check rate cap? */
@@ -642,6 +681,12 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct
ibv_ah_attr *attr)
memcpy(ah->av.dgid, attr->grh.dgid.raw, 16);
}
+ if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET)
+ if (mlx4_resolve_grh_to_l2(ah, attr)) {
+ free(ah);
+ return NULL;
+ }
+
return &ah->ibv_ah;
}
diff --git a/src/wqe.h b/src/wqe.h
index 6f7f309..043f0da 100644
--- a/src/wqe.h
+++ b/src/wqe.h
@@ -78,7 +78,8 @@ struct mlx4_wqe_datagram_seg {
uint32_tav[8];
uint32_tdqpn;
uint32_tqkey;
- uint32_treserved[2];
+ uint16_treserved;
+ uint8_t mac[6];
};
struct mlx4_wqe_data_seg {
--
1.5.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 6/7] libmlx4: add IBoE UD/VLANs support
Add VLANs support for the UD address handle creation flow,
where the vlan id is taken from the destination GID and the
and vlan priority from the IB SL specified by the application.
Signed-off-by: Or Gerlitz
---
src/mlx4.h |1 +
src/qp.c|1 +
src/verbs.c |9 +++--
src/wqe.h |2 +-
4 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/src/mlx4.h b/src/mlx4.h
index b277b06..0ad838d 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -241,6 +241,7 @@ struct mlx4_av {
struct mlx4_ah {
struct ibv_ah ibv_ah;
struct mlx4_av av;
+ uint16_tvlan;
uint8_t mac[6];
};
diff --git a/src/qp.c b/src/qp.c
index 4d79e38..40a6689 100644
--- a/src/qp.c
+++ b/src/qp.c
@@ -144,6 +144,7 @@ static void set_datagram_seg(struct mlx4_wqe_datagram_seg
*dseg,
memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
dseg->dqpn = htonl(wr->wr.ud.remote_qpn);
dseg->qkey = htonl(wr->wr.ud.remote_qkey);
+ dseg->vlan = htons(to_mah(wr->wr.ud.ah)->vlan);
memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6);
}
diff --git a/src/verbs.c b/src/verbs.c
index 6620ac2..8aa9860 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -634,13 +634,18 @@ static uint16_t get_vlan_id(union ibv_gid *gid)
static int mlx4_resolve_grh_to_l2(struct mlx4_ah *ah, struct ibv_ah_attr *attr)
{
- if (get_vlan_id(&attr->grh.dgid) != 0x)
- return 1;
+ uint16_t vid;
if (link_local_gid(&attr->grh.dgid)) {
memcpy(ah->mac, &attr->grh.dgid.raw[8], 3);
memcpy(ah->mac + 3, &attr->grh.dgid.raw[13], 3);
ah->mac[0] ^= 2;
+
+ vid = get_vlan_id(&attr->grh.dgid);
+ if (vid != 0x) {
+ ah->av.port_pd |= htonl(1 << 29);
+ ah->vlan = vid | ((attr->sl & 7) << 13);
+ }
return 0;
} else
return 1;
diff --git a/src/wqe.h b/src/wqe.h
index 043f0da..bbd22ba 100644
--- a/src/wqe.h
+++ b/src/wqe.h
@@ -78,7 +78,7 @@ struct mlx4_wqe_datagram_seg {
uint32_tav[8];
uint32_tdqpn;
uint32_tqkey;
- uint16_treserved;
+ uint16_tvlan;
uint8_t mac[6];
};
--
1.5.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 7/7] libmlx4: align the list of ConnectX devices supported with the kernel
align the list of ConnectX devices supported by the library to be
the same as the mlx4 driver from the upstream kernel
Signed-off-by: Or Gerlitz
---
Roland, these two simple awk/cut commands can be used to
actually validate the claim made by the change log...
$ grep MELLANOX libmlx4.git/src/mlx4.c | grep HCA | awk '{ print $2 }' | cut -d
")" -f 1 > lib
$ grep MELLANOX linux-2.6.git/drivers/net/mlx4/main.c | awk '{ print $3 }' |
cut -d ")" -f 1 > ker
$ diff lib ker
The patch previously posted @
http://www.spinics.net/lists/linux-rdma/msg07237.html
contains also the VPI/DDR device which isn't present in the kernel (device id
0x6778).
Once the table cleanup brought by this patch takes place, we can add further
patches to both the library and the kernel e.g with that DDR device, etc.
src/mlx4.c | 23 +++
1 files changed, 23 insertions(+), 0 deletions(-)
diff --git a/src/mlx4.c b/src/mlx4.c
index 1295c53..ab18b4a 100644
--- a/src/mlx4.c
+++ b/src/mlx4.c
@@ -66,8 +66,31 @@ struct {
HCA(MELLANOX, 0x6354), /* MT25408 "Hermon" QDR */
HCA(MELLANOX, 0x6732), /* MT25408 "Hermon" DDR PCIe gen2 */
HCA(MELLANOX, 0x673c), /* MT25408 "Hermon" QDR PCIe gen2 */
+ HCA(MELLANOX, 0x6368), /* MT25408 "Hermon" EN 10GigE */
+ HCA(MELLANOX, 0x6750), /* MT25408 "Hermon" EN 10GigE PCIe gen2 */
+ HCA(MELLANOX, 0x6372), /* MT25458 ConnectX EN 10GBASE-T 10GigE */
+ HCA(MELLANOX, 0x675a), /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */
+ HCA(MELLANOX, 0x6764), /* MT26468 ConnectX EN 10GigE PCIe gen2*/
+ HCA(MELLANOX, 0x6746), /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */
+ HCA(MELLANOX, 0x676e), /* MT26478 ConnectX2 40GigE PCIe gen2 */
+ HCA(MELLANOX, 0x1002), /* MT25400 Family [ConnectX-2 Virtual Function]
*/
+ HCA(MELLANOX, 0x1003), /* MT27500 Family [ConnectX-3] */
+ HCA(MELLANOX, 0x1004), /* MT27500 Family [ConnectX-3 Virtual Function]
*/
+ HCA(MELLANOX, 0x1005), /* MT27510 Family */
+ HCA(MELLANOX, 0x1006), /* MT27511 Family */
+ HCA(MELLANOX, 0x1007), /* MT27520 Family */
+ HCA(MELLANOX, 0x1008), /* MT27521 Family */
+ HCA(MELLANOX, 0x1009), /* MT27530 Family */
+ HCA(MELLANOX, 0x100a), /* MT27531 Family */
+ HCA(MELLANOX, 0x100b), /* MT27540 Family */
+ HCA(MELLANOX, 0x100c), /* MT27541 Family */
+ HCA(MELLANOX, 0x100d), /* MT27550 Family */
+ HCA(MELLANOX, 0x100e), /* MT27551 Family */
+ HCA(MELLANOX, 0x100f), /* MT27560 Family */
+ HCA(MELLANOX, 0x1010), /* MT27561 Family */
};
+
static struct ibv_context_ops mlx4_ctx_ops = {
.query_device = mlx4_query_device,
.query_port= mlx4_query_port,
--
1.5.5
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] ib_srp: Avoid that LUN scanning creates duplicate devices
On Sat, Jun 25, 2011 at 6:14 PM, David Dillow wrote: > On Sat, 2011-06-25 at 10:10 +0200, Bart Van Assche wrote: > > SCSI scanning of a channel:id:lun triplet in Linux works as follows > > (function > > scsi_scan_target() in drivers/scsi/scsi_scan.c): > > - If lun == SCAN_WILD_CARD, send a REPORT LUNS command to the target and > > process the result. > > - If lun != SCAN_WILD_CARD, send an INQUIRY command to the LUN corresponding > > to the specified channel:id:lun triplet to verify whether the LUN exists. > > So a SCSI driver must either take the channel and target id values in > > account > > in its quecommand() function or it should declare that it only supports one > > channel and one target id. Currently the ib_srp driver does neither. As a > > result scanning the SCSI bus via e.g. rescan-scsi-bus.sh causes many > > duplicate SCSI devices to be created. For each 0:0:L device, several > > duplicates are created with the same LUN number and with (C:I) != (0:0). Fix > > this by declaring that the ib_srp driver only supports one channel and one > > target id. > > > > Signed-off-by: Bart Van Assche > > Cc: David Dillow > > Thanks for the additional detail in the commit message. > > Acked-by: David Dillow Hi Dave, Thanks for acking. Through which tree should this patch go in ? Bart. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] ib_srp: Avoid that LUN scanning creates duplicate devices
On Tue, Jul 19, 2011 at 9:03 AM, David Dillow wrote: > I Ack'd thinking Roland would pick it up directly since has done that in > the past for singe patches, but I'm fine with pulling it into my tree > and pushing that way. > > Roland? I've had it in my for-next branch for a while now... since it wasn't a regression (AFAICT, we've been that way approximately forever), I was waiting for 3.1 to ask Linus to take it. - R. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 5/7] libmlx4: add IBoE support
On Tue, Jul 19, 2011 at 12:32:52PM +0300, Or Gerlitz wrote: > Modify libmlx4 to support IBoE, where the only user space piece > to handle is the creation of UD address handles - the L2 Ethernet > attributes have to be resolved from the DGID. It seems a shame to add a kernel syscall to every ah creation just to learn the link type. Other than rejecting some GIDs it doesn't seem like there is any operational need to learn the link type.. Also, did anyone document the GID format for IBoE? IIRC we invented this for Linux since IBA refused to standardize anything. Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 4/7] libibverbs: Update examples for IBoE
On Tue, Jul 19, 2011 at 12:31:32PM +0300, Or Gerlitz wrote: > Since IBoE requires usage of GRH, update ibv_*_pinpong examples to accept > GIDs. GIDs are given as an index to the local port's table and are exchanged > between the client and the server through the socket connection. Since these examples are meant as programming examples, using the GID index as an command line parameter is not an example of good UI design. Other tools (ie diags, ibtool, etc) accept the GID in canonical IPv6 format and match that to the GID table. Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] ib_srp: Avoid that LUN scanning creates duplicate devices
On Tue, 2011-07-19 at 07:24 -0400, Bart Van Assche wrote: > On Sat, Jun 25, 2011 at 6:14 PM, David Dillow wrote: > > On Sat, 2011-06-25 at 10:10 +0200, Bart Van Assche wrote: > > > So a SCSI driver must either take the channel and target id values in > > > account > > > in its quecommand() function or it should declare that it only supports > > > one > > > channel and one target id. Currently the ib_srp driver does neither. As a > > > result scanning the SCSI bus via e.g. rescan-scsi-bus.sh causes many > > > duplicate SCSI devices to be created. For each 0:0:L device, several > > > duplicates are created with the same LUN number and with (C:I) != (0:0). > > > Fix > > > this by declaring that the ib_srp driver only supports one channel and one > > > target id. > > > > > > Signed-off-by: Bart Van Assche > > > Cc: David Dillow > > > > Thanks for the additional detail in the commit message. > > > > Acked-by: David Dillow > > Hi Dave, > > Thanks for acking. Through which tree should this patch go in ? I Ack'd thinking Roland would pick it up directly since has done that in the past for singe patches, but I'm fine with pulling it into my tree and pushing that way. Roland? -- Dave Dillow National Center for Computational Science Oak Ridge National Laboratory (865) 241-6602 office -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 3/7] libibverbs: add GID change event
On Tue, Jul 19, 2011 at 12:30:08PM +0300, Or Gerlitz wrote: > Add IB GID change event which is generated by the kernel > IBoE stack when the HW driver updates the GID table. You also need to update ibv_event_type_str Has a kernel patch that reserves/uses the corresponding ID been accepted yet? BTW, this has value for straight IB, hopefully it triggers on IB when the GUID table is altered and when the subnet prefix is changed... Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/7] libibverbs: Add link layer field port attribute
On Tue, Jul 19, 2011 at 12:27:17PM +0300, Or Gerlitz wrote: > The addition of the new field does not change the size of struct > ibv_port_attr due to alignment of the preceding fields. As such > binary compatibility between the library to applications is kept, What zeros the field in the current library? It looks like new apps that want to access the link_layer field will have to explicitly zero port_attr before calling ibv_cmd_query_port. Should be noted in the documentation - and maybe also explicitly zero the remaining pad for future. Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 5/7] libmlx4: add IBoE support
Jason Gunthorpe wrote: >> Modify libmlx4 to support IBoE, where the only user space piece >> to handle is the creation of UD address handles - the L2 Ethernet >> attributes have to be resolved from the DGID. > Other than rejecting some GIDs it doesn't seem like there is any > operational need to learn the link type.. Jason, as the change log explains, under Ethernet link type the code (see mlx4_resolve_grh_to_l2) goes and extracts the Ethernet L2 info (mac and vlan, look on the last patch) from the GID. > It seems a shame to add a kernel syscall to every ah creation just to learn > the link type AH creation is app's slow path, e.g the equivalent of session creation, so I wouldn't bother too much on perf, on the being elegant side, I'll check if/how I can plant/cache the link type for the ah creation code to use. > Also, did anyone document the GID format for IBoE? IIRC we invented > this for Linux since IBA refused to standardize anything. Not really, maybe we can/should add a document to the kernel IB documentation folder with some details - as this data is more for deep divers - the thing is that applications that use the rdma-cm don't deal directly with the GIDs, they just get it from the kernel and hand it over to libibverbs (e.g UD apps use address / route resolution and then call ah create, RC apps even don't have to call libibiverbs, as the qp modifications go beyond the cover of librdmacm calling libibverbs). Or. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/7] libibverbs: Add link layer field port attribute
Jason Gunthorpe wrote: > What zeros the field in the current library? the kernel uverbs code either zeros it (old kernels) or sets it (new kernels) Or. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 3/7] libibverbs: add GID change event
Jason Gunthorpe wrote: > You also need to update ibv_event_type_str sure, will do that. > Has a kernel patch that reserves/uses the corresponding ID been accepted yet? yes, its in Roland's for-next branch > BTW, this has value for straight IB, hopefully it triggers on IB > when the GUID table is altered and when the subnet prefix is changed... The kernel is triggering this event when the GID table is changed from the IBoE code path, the library is just receiving the event and delivering it over to the application. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 4/7] libibverbs: Update examples for IBoE
Jason Gunthorpe wrote: >> Since IBoE requires usage of GRH, update ibv_*_pinpong examples to accept >> GIDs. GIDs are given as an index to the local port's table and are exchanged >> between the client and the server through the socket connection. > Since these examples are meant as programming examples, using the GID > index as an command line parameter is not an example of good UI > design. Other tools (ie diags, ibtool, etc) accept the GID in > canonical IPv6 format and match that to the GID table. To my taste these examples should be used as programming examples for their verbs data path part, as for connection establishment, I would prefer app writers use the rdma-cm and I don't think the control plane of these examples (the IB L2/L3/L4 info exchange) should be used in real life apps. In the same manner that under IB the user doesn't specify the LID from the command line but rather the device and port, I don't think they need to specify the GID but rather the device and port. The index isn't always zero, since this is what let people test VLANs (under IBoE for each VLAN a new entry is added to the GID table as the GID includes both mac and vlan). Or. Or. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/7] libibverbs: Add link layer field port attribute
On Tue, Jul 19, 2011 at 10:36:26PM +0300, Or Gerlitz wrote: > > What zeros the field in the current library? > > the kernel uverbs code either zeros it (old kernels) or sets it (new kernels) ibv_cmd_query_port does a field by field copy of the port_attr, so what the kernel does is not relevant to this case, and to me it looks like the new value you added will be left uninitialized on old ibverbs. Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 5/7] libmlx4: add IBoE support
Jason Gunthorpe wrote:
> Or Gerlitz wrote:
>> Jason, as the change log explains, under Ethernet link type the code
>> (see mlx4_resolve_grh_to_l2) goes and extracts the Ethernet L2 info
>> (mac and vlan, look on the last patch) from the GID.
>
> Looking at the control flow of the actual code, there is no reason it
> needs to know the link type, except to generate that error code. Why
> can't it set the slid, dlid and MAC fields in mlx4_ah simultaneously?
If you look on patch 6/7 @ mlx4_resolve_grh_to_l2, you would see this section
+ if (vid != 0x) {
+ ah->av.port_pd |= htonl(1 << 29);
+ ah->vlan = vid | ((attr->sl & 7) << 13);
+ }
this bit in the PD tells the HW to add VLAN header to the packet and
sets the user priority bits for that vlan from the SL, we want to do
that only for Ethernet link type.
>> AH creation is app's slow path, e.g the equivalent of session
>> creation, so I wouldn't bother too much on perf, on the being elegant
>> side, I'll check if/how I can plant/cache the link type for the ah
>> creation code to use.
> Still, the MPI people regularly seem to have job startup problems, no
> idea how many AH's they end up creating at startup though..
Under IB job startup problems relates to SA scalability, generally under any
transport @ large scale there could be more issues, I don't think AH
creation time
is among them.
Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/7] libibverbs: Add link layer field port attribute
Jason Gunthorpe wrote: > Or Gerlitz wrote: >>> What zeros the field in the current library? + port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED; + port_attr->pad = 0; Or -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/7] libibverbs: Add link layer field port attribute
On Tue, Jul 19, 2011 at 11:22:51PM +0300, Or Gerlitz wrote: > Jason Gunthorpe wrote: > > Or Gerlitz wrote: > > >>> What zeros the field in the current library? > > + port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED; > + port_attr->pad = 0; Where are you getting that hunk from? libibverbs $ git grep link_layer libibverbs-1.1.5 -- libibverbs $ More clearly: what initializes the memory you've redefined to be link_layer in libibverbs 1.1.5 ? I see nothing. Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
RE: [PATCH 1/7] libibverbs: Add link layer field port attribute
> More clearly: what initializes the memory you've redefined to be > link_layer in libibverbs 1.1.5 ? I see nothing. Why does the value in an older library cause an issue? Or, how will this library version interact with an older kernel? -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/7] libibverbs: Add link layer field port attribute
On Tue, Jul 19, 2011 at 09:20:25PM +, Hefty, Sean wrote:
> > More clearly: what initializes the memory you've redefined to be
> > link_layer in libibverbs 1.1.5 ? I see nothing.
>
> Why does the value in an older library cause an issue?
ABI compatability means if dynamic linking succeeds then the result
works, so the behavior of a new app, using new functionality when it
links to the old library must work.
The commit comment states the goal is to have new apps see
IBV_LINK_LAYER_UNSPECIFIED for all possible combinations of old stuff
underneath.
> Or, how will this library version interact with an older kernel?
The kernel interaction is not what I'm talking about - AFAIK, the
kernel interaction is OK. Old kernels zero the entire ib_port_attr
structure before copy_to_user so the reserved bytes are guaranteed to
be 0.
The problem is with the userspace ABI out of libibverbs. Current
libibverbs does not zero its ib_port_attr structure before returning.
Build this with Or's patch applied, then run it against libibverbs 1.1.5
struct ibv_port_attr attr;
attr.link_layer = 100;
ibv_query_port(..,&attr);
assert(attr.link_layer == IBV_LINK_LAYER_UNSPECIFIED); // Fails!
That is an ABI breaking issue that is not dealt with by Or's patch.
Couple choices:
1) Require and document callers using ibv_query_port to zero attr
before hand when using link_layer
2) Crank the symbol version on ibv_query_port so dynamic linking
fails
3) Inline the memset in ibverbs.h:
static inline int fixed_ibv_query_port(...)
{
attr->link_layer = 0;
return ibv_query_port(...);
}
#define ibv_query_port(...) fixed_ibv_query_port(...)
#2 might be the best?
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[infiniband-diags] [libibmad] Support optional performance counters
The following patches adds support of many of the infiniband spec's
optional performance counters into libibmad and infiniband-diags'
perfquery.
Al
--
Albert Chu
ch...@llnl.gov
Computer Scientist
High Performance Systems Division
Lawrence Livermore National Laboratory
--- Begin Message ---
Signed-off-by: Albert L. Chu
---
man/perfquery.8 | 46 -
src/perfquery.c | 197 ++-
2 files changed, 238 insertions(+), 5 deletions(-)
diff --git a/man/perfquery.8 b/man/perfquery.8
index e01dc2f..7acc60c 100644
--- a/man/perfquery.8
+++ b/man/perfquery.8
@@ -6,10 +6,12 @@ perfquery \- query InfiniBand port counters
.SH SYNOPSIS
.B perfquery
[\-d(ebug)] [\-G(uid)] [\-x|\-\-extended] [\-X|\-\-xmtsl] [\-S|\-\-rcvsl]
-[\-D|\-\-xmtdisc] [\-E|\-\-rcverr] [\-c|\-\-smplctl]
-[-a(ll_ports)] [-l(oop_ports)] [-r(eset_after_read)] [-R(eset_only)]
-[\-C ca_name] [\-P ca_port] [\-t(imeout) timeout_ms] [\-V(ersion)] [\-h(elp)]
-[ [[port] [reset_mask]]]
+[\-D|\-\-xmtdisc] [\-E|\-\-rcverr] [\-\-oprcvcounters] [\-\-flowctlcounters]
+[\-\-vloppackets] [\-\-vlopdata] [\-\-vlxmitflowctlerrors] [\-\-vlxmitcounters]
+[\-\-swportvlcong] [\-\-rcvcc] [\-\-slrcvfecn] [\-\-slrcvbecn] [\-\-xmitcc]
+[\-\-vlxmittimecc] [\-c|\-\-smplctl] [-a(ll_ports)] [-l(oop_ports)]
+[-r(eset_after_read)] [-R(eset_only)] [\-C ca_name] [\-P ca_port] [\-t(imeout)
timeout_ms]
+[\-V(ersion)] [\-h(elp)] [ [[port] [reset_mask]]]
.SH DESCRIPTION
.PP
@@ -49,6 +51,42 @@ show receive error details. This is an optional counter.
\fB\-D\fR, \fB\-\-xmtdisc\fR
show transmit discard details. This is an optional counter.
.TP
+\fB\-\-oprcvcounters\fR
+show Rcv Counters per Op code. This is an optional counter.
+.TP
+\fB\-\-flowctlcounters\fR
+show flow control counters. This is an optional counter.
+.TP
+\fB\-\-vloppackets\fR
+show packets received per Op code per VL. This is an optional counter.
+.TP
+\fB\-\-vlopdata\fR
+show data received per Op code per VL. This is an optional counter.
+.TP
+\fB\-\-vlxmitflowctlerrors\fR
+show flow control update errors per VL. This is an optional counter.
+.TP
+\fB\-\-vlxmitcounters\fR
+show ticks waiting to transmit counters per VL. This is an optional counter.
+.TP
+\fB\-\-swportvlcong\fR
+show sw port VL congestion. This is an optional counter.
+.TP
+\fB\-\-rcvcc\fR
+show Rcv congestion control counters. This is an optional counter.
+.TP
+\fB\-\-slrcvfecn\fR
+show SL Rcv FECN counters. This is an optional counter.
+.TP
+\fB\-\-slrcvbecn\fR
+show SL Rcv BECN counters. This is an optional counter.
+.TP
+\fB\-\-xmitcc\fR
+show Xmit congestion control counters. This is an optional counter.
+.TP
+\fB\-\-vlxmittimecc\fR
+show VL Xmit Time congestion control counters. This is an optional counter.
+.TP
\fB\-c\fR, \fB\-\-smplctl\fR
show port samples control.
.TP
diff --git a/src/perfquery.c b/src/perfquery.c
index 8923654..e0dbf5a 100644
--- a/src/perfquery.c
+++ b/src/perfquery.c
@@ -368,7 +368,9 @@ static void reset_counters(int extended, int timeout, int
mask,
}
static int reset, reset_only, all_ports, loop_ports, port, extended, xmt_sl,
-rcv_sl, xmt_disc, rcv_err, smpl_ctl;
+rcv_sl, xmt_disc, rcv_err, smpl_ctl, oprcvcounters, flowctlcounters,
+vloppackets, vlopdata, vlxmitflowctlerrors, vlxmitcounters, swportvlcong,
+rcvcc, slrcvfecn, slrcvbecn, xmitcc, vlxmittimecc;
static void common_func(ib_portid_t * portid, int port_num, int mask,
unsigned query, unsigned reset,
@@ -423,6 +425,90 @@ static void rcv_err_query(ib_portid_t * portid, int port,
int mask)
mad_dump_perfcounters_rcv_err);
}
+static void oprcvcounters_query(ib_portid_t * portid, int port, int mask)
+{
+ common_func(portid, port, mask, !reset_only, (reset_only || reset),
+ "PortOpRcvCounters", IB_GIS_PORT_PORT_OP_RCV_COUNTERS,
+ mad_dump_perfcounters_port_op_rcv_counters);
+}
+
+static void flowctlcounters_query(ib_portid_t * portid, int port, int mask)
+{
+ common_func(portid, port, mask, !reset_only, (reset_only || reset),
+ "PortFlowCtlCounters", IB_GIS_PORT_PORT_FLOW_CTL_COUNTERS,
+ mad_dump_perfcounters_port_flow_ctl_counters);
+}
+
+static void vloppackets_query(ib_portid_t * portid, int port, int mask)
+{
+ common_func(portid, port, mask, !reset_only, (reset_only || reset),
+ "PortVLOpPackets", IB_GIS_PORT_PORT_VL_OP_PACKETS,
+ mad_dump_perfcounters_port_vl_op_packet);
+}
+
+static void vlopdata_query(ib_portid_t * portid, int port, int mask)
+{
+ common_func(portid, port, mask, !reset_only, (reset_only || reset),
+ "PortVLOpData", IB_GIS_PORT_PORT_VL_OP_DATA,
+ mad_dump_perfcounters_port_vl_op_data);
+}
+
+static void vlxmitflowctlerrors_query(ib_portid_t * portid, int port, int mask)
+{
+ common_func(portid, port, mask, !reset_only, (reset_only || re
