[PATCH 25/30] IB/mlx4: Configure device to work in RoCEv2
From: Moni Shoua mo...@mellanox.com
Some mlx4 adapters are RoCEv2 capable. To enable this feature some
hardware configuration is required. This is
1. Set port general parameters
2. Configure the outgoing UDP destination port
3. Configure the QP that work with RoCEv2
Signed-off-by: Moni Shoua mo...@mellanox.com
Signed-off-by: Somnath Kotur somnath.ko...@emulex.com
---
drivers/infiniband/hw/mlx4/main.c | 10 ++-
drivers/infiniband/hw/mlx4/qp.c | 39 ++--
drivers/net/ethernet/mellanox/mlx4/fw.c | 16 +++-
drivers/net/ethernet/mellanox/mlx4/mlx4.h |3 +-
drivers/net/ethernet/mellanox/mlx4/port.c |9 ++-
drivers/net/ethernet/mellanox/mlx4/qp.c | 27
include/linux/mlx4/device.h |3 +-
include/linux/mlx4/qp.h | 15 +-
8 files changed, 112 insertions(+), 10 deletions(-)
diff --git a/drivers/infiniband/hw/mlx4/main.c
b/drivers/infiniband/hw/mlx4/main.c
index ca19d1d..50612b8 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -2154,7 +2154,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
if (mlx4_ib_init_sriov(ibdev))
goto err_mad;
- if (dev-caps.flags MLX4_DEV_CAP_FLAG_IBOE) {
+ if (dev-caps.flags MLX4_DEV_CAP_FLAG_IBOE ||
+ dev-caps.flags2 MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
if (!iboe-nb.notifier_call) {
iboe-nb.notifier_call = mlx4_ib_netdev_event;
err = register_netdevice_notifier(iboe-nb);
@@ -2163,6 +2164,13 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
goto err_notif;
}
}
+ if (!mlx4_is_slave(dev)
+ dev-caps.flags2 MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
+ err = mlx4_config_roce_v2_port(dev, ROCE_V2_UDP_DPORT);
+ if (err) {
+ goto err_notif;
+ }
+ }
}
for (j = 0; j ARRAY_SIZE(mlx4_class_attributes); ++j) {
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 9ab9156..9731c07 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -1408,6 +1408,24 @@ static int handle_eth_ud_smac_index(struct mlx4_ib_dev
*dev,
return 0;
}
+enum {
+ MLX4_QPC_ROCE_MODE_1 = 0,
+ MLX4_QPC_ROCE_MODE_2 = 2,
+ MLX4_QPC_ROCE_MODE_MAX = 0xff
+};
+
+static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
+{
+ switch (gid_type) {
+ case IB_GID_TYPE_IB:
+ return MLX4_QPC_ROCE_MODE_1;
+ case IB_GID_TYPE_ROCE_V2:
+ return MLX4_QPC_ROCE_MODE_2;
+ default:
+ return MLX4_QPC_ROCE_MODE_MAX;
+ }
+}
+
static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
const struct ib_qp_attr *attr, int attr_mask,
enum ib_qp_state cur_state, enum ib_qp_state
new_state)
@@ -1532,9 +1550,12 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
u16 vlan = 0x;
u8 smac[ETH_ALEN];
int status = 0;
+ int is_eth = rdma_port_get_link_layer(dev-ib_dev, qp-port) ==
+ IB_LINK_LAYER_ETHERNET;
- if (rdma_port_get_link_layer(dev-ib_dev, qp-port) ==
- IB_LINK_LAYER_ETHERNET) {
+ if (is_eth) {
+ if (mlx4_is_bonded(dev-dev))
+ port_num = 1;
rcu_read_lock();
status = ib_get_cached_gid(ibqp-device, port_num,
index, gid, gid_attr);
@@ -1551,8 +1572,20 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
port_num, vlan, smac))
goto out;
+ if (is_eth gid_attr.gid_type == IB_GID_TYPE_ROCE_V2)
+ context-pri_path.hop_limit = IPV6_DEFAULT_HOPLIMIT;
+
optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
MLX4_QP_OPTPAR_SCHED_QUEUE);
+
+ if (is_eth (cur_state == IB_QPS_INIT new_state ==
IB_QPS_RTR)) {
+ u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
+
+ if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_MAX)
+ goto out;
+ context-rlkey_roce_mode |= (qpc_roce_mode 6);
+ }
+
}
if (attr_mask IB_QP_TIMEOUT) {
@@ -1722,7 +1755,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
sqd_event = 0;
if (!ibqp-uobject cur_state == IB_QPS_RESET new_state ==
IB_QPS_INIT)
- context-rlkey |= (1 4);
+ context-rlkey_roce_mode |=
[PATCH 25/30] IB/mlx4: Configure device to work in RoCEv2
From: Moni Shoua mo...@mellanox.com
Some mlx4 adapters are RoCEv2 capable. To enable this feature some
hardware configuration is required. This is
1. Set port general parameters
2. Configure the outgoing UDP destination port
3. Configure the QP that work with RoCEv2
Signed-off-by: Moni Shoua mo...@mellanox.com
Signed-off-by: Somnath Kotur somnath.ko...@emulex.com
---
drivers/infiniband/hw/mlx4/main.c | 10 ++-
drivers/infiniband/hw/mlx4/qp.c | 39 ++--
drivers/net/ethernet/mellanox/mlx4/fw.c | 16 +++-
drivers/net/ethernet/mellanox/mlx4/mlx4.h |3 +-
drivers/net/ethernet/mellanox/mlx4/port.c |9 ++-
drivers/net/ethernet/mellanox/mlx4/qp.c | 27
include/linux/mlx4/device.h |3 +-
include/linux/mlx4/qp.h | 15 +-
8 files changed, 112 insertions(+), 10 deletions(-)
diff --git a/drivers/infiniband/hw/mlx4/main.c
b/drivers/infiniband/hw/mlx4/main.c
index ca19d1d..50612b8 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -2154,7 +2154,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
if (mlx4_ib_init_sriov(ibdev))
goto err_mad;
- if (dev-caps.flags MLX4_DEV_CAP_FLAG_IBOE) {
+ if (dev-caps.flags MLX4_DEV_CAP_FLAG_IBOE ||
+ dev-caps.flags2 MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
if (!iboe-nb.notifier_call) {
iboe-nb.notifier_call = mlx4_ib_netdev_event;
err = register_netdevice_notifier(iboe-nb);
@@ -2163,6 +2164,13 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
goto err_notif;
}
}
+ if (!mlx4_is_slave(dev)
+ dev-caps.flags2 MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
+ err = mlx4_config_roce_v2_port(dev, ROCE_V2_UDP_DPORT);
+ if (err) {
+ goto err_notif;
+ }
+ }
}
for (j = 0; j ARRAY_SIZE(mlx4_class_attributes); ++j) {
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 9ab9156..9731c07 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -1408,6 +1408,24 @@ static int handle_eth_ud_smac_index(struct mlx4_ib_dev
*dev,
return 0;
}
+enum {
+ MLX4_QPC_ROCE_MODE_1 = 0,
+ MLX4_QPC_ROCE_MODE_2 = 2,
+ MLX4_QPC_ROCE_MODE_MAX = 0xff
+};
+
+static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
+{
+ switch (gid_type) {
+ case IB_GID_TYPE_IB:
+ return MLX4_QPC_ROCE_MODE_1;
+ case IB_GID_TYPE_ROCE_V2:
+ return MLX4_QPC_ROCE_MODE_2;
+ default:
+ return MLX4_QPC_ROCE_MODE_MAX;
+ }
+}
+
static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
const struct ib_qp_attr *attr, int attr_mask,
enum ib_qp_state cur_state, enum ib_qp_state
new_state)
@@ -1532,9 +1550,12 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
u16 vlan = 0x;
u8 smac[ETH_ALEN];
int status = 0;
+ int is_eth = rdma_port_get_link_layer(dev-ib_dev, qp-port) ==
+ IB_LINK_LAYER_ETHERNET;
- if (rdma_port_get_link_layer(dev-ib_dev, qp-port) ==
- IB_LINK_LAYER_ETHERNET) {
+ if (is_eth) {
+ if (mlx4_is_bonded(dev-dev))
+ port_num = 1;
rcu_read_lock();
status = ib_get_cached_gid(ibqp-device, port_num,
index, gid, gid_attr);
@@ -1551,8 +1572,20 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
port_num, vlan, smac))
goto out;
+ if (is_eth gid_attr.gid_type == IB_GID_TYPE_ROCE_V2)
+ context-pri_path.hop_limit = IPV6_DEFAULT_HOPLIMIT;
+
optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
MLX4_QP_OPTPAR_SCHED_QUEUE);
+
+ if (is_eth (cur_state == IB_QPS_INIT new_state ==
IB_QPS_RTR)) {
+ u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
+
+ if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_MAX)
+ goto out;
+ context-rlkey_roce_mode |= (qpc_roce_mode 6);
+ }
+
}
if (attr_mask IB_QP_TIMEOUT) {
@@ -1722,7 +1755,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
sqd_event = 0;
if (!ibqp-uobject cur_state == IB_QPS_RESET new_state ==
IB_QPS_INIT)
- context-rlkey |= (1 4);
+ context-rlkey_roce_mode |=
