mlx4: limit allowable qp create resources to avoid create_qp failures
due to added headroom wqes.
In addition, guarantee that qp capabilities following qp creation
always lie within limits given by ib_query_device.
(for userspace, we perform this limiting in libmlx4, so as not to
break the ABI).
Signed-off-by: Jack Morgenstein <[EMAIL PROTECTED]>
diff --git a/drivers/infiniband/hw/mlx4/main.c
b/drivers/infiniband/hw/mlx4/main.c
index d8287d9..d40ec2f 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -109,7 +109,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
props->max_mr_size = ~0ull;
props->page_size_cap = dev->dev->caps.page_size_cap;
props->max_qp = dev->dev->caps.num_qps -
dev->dev->caps.reserved_qps;
- props->max_qp_wr = dev->dev->caps.max_wqes;
+ props->max_qp_wr = dev->dev->caps.max_wqes -
MLX4_IB_SQ_MAX_SPARE;
props->max_sge = min(dev->dev->caps.max_sq_sg,
dev->dev->caps.max_rq_sg);
props->max_cq = dev->dev->caps.num_cqs -
dev->dev->caps.reserved_cqs;
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h
b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 2869765..56305e2 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -47,6 +47,13 @@ enum {
MLX4_IB_DB_PER_PAGE = PAGE_SIZE / 4
};
+enum {
+ MLX4_IB_SQ_MIN_WQE_SHIFT = 6
+};
+
+#define MLX4_IB_SQ_HEADROOM(shift) ((2048 >> (shift)) + 1)
+#define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT))
+
struct mlx4_ib_db_pgdir;
struct mlx4_ib_user_db_page;
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 6b33224..d6c1600 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -212,8 +212,9 @@ static int set_rq_size(struct mlx4_ib_dev *dev, struct
ib_qp_cap *cap,
int is_user, int has_srq, struct mlx4_ib_qp *qp)
{
/* Sanity check RQ size before proceeding */
- if (cap->max_recv_wr > dev->dev->caps.max_wqes ||
- cap->max_recv_sge > dev->dev->caps.max_rq_sg)
+ if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
+ cap->max_recv_sge >
+ min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
return -EINVAL;
if (has_srq) {
@@ -232,8 +233,19 @@ static int set_rq_size(struct mlx4_ib_dev *dev, struct
ib_qp_cap *cap,
qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct
mlx4_wqe_data_seg));
}
- cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
- cap->max_recv_sge = qp->rq.max_gs;
+ /* leave userspace return values as they were, so as not to break ABI */
+ if (is_user) {
+ cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
+ cap->max_recv_sge = qp->rq.max_gs;
+ } else {
+ cap->max_recv_wr = qp->rq.max_post =
+ min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE,
qp->rq.wqe_cnt);
+ cap->max_recv_sge = min(qp->rq.max_gs,
+ min(dev->dev->caps.max_sq_sg,
+ dev->dev->caps.max_rq_sg));
+ }
+ /* We don't support inline sends for kernel QPs (yet) */
+
return 0;
}
@@ -242,8 +254,9 @@ static int set_kernel_sq_size(struct mlx4_ib_dev *dev,
struct ib_qp_cap *cap,
enum ib_qp_type type, struct mlx4_ib_qp *qp)
{
/* Sanity check SQ size before proceeding */
- if (cap->max_send_wr > dev->dev->caps.max_wqes ||
- cap->max_send_sge > dev->dev->caps.max_sq_sg ||
+ if (cap->max_send_wr > (dev->dev->caps.max_wqes -
MLX4_IB_SQ_MAX_SPARE) ||
+ cap->max_send_sge >
+ min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
cap->max_inline_data + send_wqe_overhead(type) +
sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
return -EINVAL;
@@ -261,6 +274,7 @@ static int set_kernel_sq_size(struct mlx4_ib_dev *dev,
struct ib_qp_cap *cap,
cap->max_inline_data +
sizeof (struct
mlx4_wqe_inline_seg)) +
send_wqe_overhead(type)));
+ qp->sq.wqe_shift = max(MLX4_IB_SQ_MIN_WQE_SHIFT, qp->sq.wqe_shift);
qp->sq.max_gs = ((1 << qp->sq.wqe_shift) - send_wqe_overhead(type)) /
sizeof (struct mlx4_wqe_data_seg);
@@ -268,7 +282,7 @@ static int set_kernel_sq_size(struct mlx4_ib_dev *dev,
struct ib_qp_cap *cap,
* We need to leave 2 KB + 1 WQE of headroom in the SQ to
* allow HW to prefetch.
*/
- qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
+ qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);
qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr +
qp->sq_spare_wqes);
qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
@@ -281,8 +295,12 @@ static int set_kernel_sq_size(struct mlx4_ib_dev *dev,
struct ib_qp_cap *cap,
qp->sq.offset = 0;
}
- cap->max_send_wr = qp->sq.max_post = qp->sq.wqe_cnt -
qp->sq_spare_wqes;
- cap->max_send_sge = qp->sq.max_gs;
+ cap->max_send_wr = qp->sq.max_post =
+ min(qp->sq.wqe_cnt - qp->sq_spare_wqes,
+ dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE);
+ cap->max_send_sge =min(qp->sq.max_gs,
+ min(dev->dev->caps.max_sq_sg,
+ dev->dev->caps.max_rq_sg));
/* We don't support inline sends for kernel QPs (yet) */
cap->max_inline_data = 0;
_______________________________________________
general mailing list
[email protected]
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
