[PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities exceeding the page size
Handle this configuration:
Queues Per Page * SGE BAR2 Queue Register Area Size Page Size
Use cxgb4_bar2_sge_qregs() to obtain the proper location within the
bar2 region for a given qid.
Rework the DB and GTS write functions to make use of this bar2 info.
Signed-off-by: Steve Wise sw...@opengridcomputing.com
Signed-off-by: Hariprasad Shenai haripra...@chelsio.com
---
drivers/infiniband/hw/cxgb4/cq.c | 22 ++--
drivers/infiniband/hw/cxgb4/device.c | 16 +++--
drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +--
drivers/infiniband/hw/cxgb4/qp.c | 64 ++
drivers/infiniband/hw/cxgb4/t4.h | 60 ---
5 files changed, 98 insertions(+), 69 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index 68ddb37..8e5bbcb 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -156,19 +156,17 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq
*cq,
goto err4;
cq-gen = 1;
+ cq-gts = rdev-lldi.gts_reg;
cq-rdev = rdev;
- if (user) {
- u32 off = (cq-cqid rdev-cqshift) PAGE_MASK;
- cq-ugts = (u64)rdev-bar2_pa + off;
- } else if (is_t4(rdev-lldi.adapter_type)) {
- cq-gts = rdev-lldi.gts_reg;
- cq-qid_mask = -1U;
- } else {
- u32 off = ((cq-cqid rdev-cqshift) PAGE_MASK) + 12;
-
- cq-gts = rdev-bar2_kva + off;
- cq-qid_mask = rdev-qpmask;
+ cq-bar2_va = c4iw_bar2_addrs(rdev, cq-cqid, T4_BAR2_QTYPE_INGRESS,
+ cq-bar2_qid,
+ user ? cq-bar2_pa : NULL);
+ if (user !cq-bar2_va) {
+ pr_warn(MOD %s: cqid %u not in BAR2 range.\n,
+ pci_name(rdev-lldi.pdev), cq-cqid);
+ ret = -EINVAL;
+ goto err4;
}
return 0;
err4:
@@ -971,7 +969,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int
entries,
insert_mmap(ucontext, mm);
mm2-key = uresp.gts_key;
- mm2-addr = chp-cq.ugts;
+ mm2-addr = (u64)(uintptr_t)chp-cq.bar2_pa;
mm2-len = PAGE_SIZE;
insert_mmap(ucontext, mm2);
}
diff --git a/drivers/infiniband/hw/cxgb4/device.c
b/drivers/infiniband/hw/cxgb4/device.c
index 1ffbd03..ab36804 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -788,13 +788,7 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
goto err1;
}
- /*
-* qpshift is the number of bits to shift the qpid left in order
-* to get the correct address of the doorbell for that qp.
-*/
- rdev-qpshift = PAGE_SHIFT - ilog2(rdev-lldi.udb_density);
rdev-qpmask = rdev-lldi.udb_density - 1;
- rdev-cqshift = PAGE_SHIFT - ilog2(rdev-lldi.ucq_density);
rdev-cqmask = rdev-lldi.ucq_density - 1;
PDBG(%s dev %s stag start 0x%0x size 0x%0x num stags %d
pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x
@@ -808,14 +802,12 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
rdev-lldi.vr-qp.size,
rdev-lldi.vr-cq.start,
rdev-lldi.vr-cq.size);
- PDBG(udb len 0x%x udb base %p db_reg %p gts_reg %p qpshift %lu
-qpmask 0x%x cqshift %lu cqmask 0x%x\n,
+ PDBG(udb len 0x%x udb base %p db_reg %p gts_reg %p
+qpmask 0x%x cqmask 0x%x\n,
(unsigned)pci_resource_len(rdev-lldi.pdev, 2),
(void *)pci_resource_start(rdev-lldi.pdev, 2),
-rdev-lldi.db_reg,
-rdev-lldi.gts_reg,
-rdev-qpshift, rdev-qpmask,
-rdev-cqshift, rdev-cqmask);
+rdev-lldi.db_reg, rdev-lldi.gts_reg,
+rdev-qpmask, rdev-cqmask);
if (c4iw_num_stags(rdev) == 0) {
err = -EINVAL;
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index d87e165..7a1b675 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -164,9 +164,7 @@ struct wr_log_entry {
struct c4iw_rdev {
struct c4iw_resource resource;
- unsigned long qpshift;
u32 qpmask;
- unsigned long cqshift;
u32 cqmask;
struct c4iw_dev_ucontext uctx;
struct gen_pool *pbl_pool;
@@ -1025,6 +1023,9 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe
*err_cqe);
extern struct cxgb4_client t4c_client;
extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
+void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
+ enum cxgb4_bar2_qtype qtype,
+ unsigned int *pbar2_qid, u64 *pbar2_pa);
extern void c4iw_log_wr_stats(struct t4_wq *wq, struct
RE: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities exceeding the page size
-Original Message-
From: Doug Ledford [mailto:dledf...@redhat.com]
Sent: Tuesday, June 09, 2015 9:03 AM
To: Hariprasad Shenai
Cc: linux-rdma@vger.kernel.org; sw...@opengridcomputing.com;
lee...@chelsio.com; nirran...@chelsio.com
Subject: Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities
exceeding the page size
On Tue, 2015-06-09 at 18:23 +0530, Hariprasad Shenai wrote:
Handle this configuration:
Queues Per Page * SGE BAR2 Queue Register Area Size Page Size
Use cxgb4_bar2_sge_qregs() to obtain the proper location within the
bar2 region for a given qid.
Rework the DB and GTS write functions to make use of this bar2 info.
Signed-off-by: Steve Wise sw...@opengridcomputing.com
Signed-off-by: Hariprasad Shenai haripra...@chelsio.com
---
drivers/infiniband/hw/cxgb4/cq.c | 22 ++--
drivers/infiniband/hw/cxgb4/device.c | 16 +++--
drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +--
drivers/infiniband/hw/cxgb4/qp.c | 64
++
drivers/infiniband/hw/cxgb4/t4.h | 60 ---
5 files changed, 98 insertions(+), 69 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cq.c
b/drivers/infiniband/hw/cxgb4/cq.c
index 68ddb37..8e5bbcb 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -156,19 +156,17 @@ static int create_cq(struct c4iw_rdev *rdev, struct
t4_cq *cq,
goto err4;
cq-gen = 1;
+ cq-gts = rdev-lldi.gts_reg;
cq-rdev = rdev;
- if (user) {
- u32 off = (cq-cqid rdev-cqshift) PAGE_MASK;
- cq-ugts = (u64)rdev-bar2_pa + off;
- } else if (is_t4(rdev-lldi.adapter_type)) {
- cq-gts = rdev-lldi.gts_reg;
- cq-qid_mask = -1U;
- } else {
- u32 off = ((cq-cqid rdev-cqshift) PAGE_MASK) + 12;
-
- cq-gts = rdev-bar2_kva + off;
- cq-qid_mask = rdev-qpmask;
+ cq-bar2_va = c4iw_bar2_addrs(rdev, cq-cqid, T4_BAR2_QTYPE_INGRESS,
+ cq-bar2_qid,
+ user ? cq-bar2_pa : NULL);
+ if (user !cq-bar2_va) {
+ pr_warn(MOD %s: cqid %u not in BAR2 range.\n,
+ pci_name(rdev-lldi.pdev), cq-cqid);
+ ret = -EINVAL;
+ goto err4;
}
return 0;
err4:
@@ -971,7 +969,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
int entries,
insert_mmap(ucontext, mm);
mm2-key = uresp.gts_key;
- mm2-addr = chp-cq.ugts;
+ mm2-addr = (u64)(uintptr_t)chp-cq.bar2_pa;
Why are you using a cast here at all? bar2_pa is already u64...
So it should just have the (uintptr_t) cast?
Steve.
--
To unsubscribe from this list: send the line unsubscribe linux-rdma in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities exceeding the page size
On Tue, 2015-06-09 at 09:07 -0500, Steve Wise wrote:
-Original Message-
From: Doug Ledford [mailto:dledf...@redhat.com]
Sent: Tuesday, June 09, 2015 9:03 AM
To: Hariprasad Shenai
Cc: linux-rdma@vger.kernel.org; sw...@opengridcomputing.com;
lee...@chelsio.com; nirran...@chelsio.com
Subject: Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities
exceeding the page size
On Tue, 2015-06-09 at 18:23 +0530, Hariprasad Shenai wrote:
Handle this configuration:
Queues Per Page * SGE BAR2 Queue Register Area Size Page Size
Use cxgb4_bar2_sge_qregs() to obtain the proper location within the
bar2 region for a given qid.
Rework the DB and GTS write functions to make use of this bar2 info.
Signed-off-by: Steve Wise sw...@opengridcomputing.com
Signed-off-by: Hariprasad Shenai haripra...@chelsio.com
---
drivers/infiniband/hw/cxgb4/cq.c | 22 ++--
drivers/infiniband/hw/cxgb4/device.c | 16 +++--
drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +--
drivers/infiniband/hw/cxgb4/qp.c | 64
++
drivers/infiniband/hw/cxgb4/t4.h | 60
---
5 files changed, 98 insertions(+), 69 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cq.c
b/drivers/infiniband/hw/cxgb4/cq.c
index 68ddb37..8e5bbcb 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -156,19 +156,17 @@ static int create_cq(struct c4iw_rdev *rdev, struct
t4_cq *cq,
goto err4;
cq-gen = 1;
+ cq-gts = rdev-lldi.gts_reg;
cq-rdev = rdev;
- if (user) {
- u32 off = (cq-cqid rdev-cqshift) PAGE_MASK;
- cq-ugts = (u64)rdev-bar2_pa + off;
- } else if (is_t4(rdev-lldi.adapter_type)) {
- cq-gts = rdev-lldi.gts_reg;
- cq-qid_mask = -1U;
- } else {
- u32 off = ((cq-cqid rdev-cqshift) PAGE_MASK) + 12;
-
- cq-gts = rdev-bar2_kva + off;
- cq-qid_mask = rdev-qpmask;
+ cq-bar2_va = c4iw_bar2_addrs(rdev, cq-cqid, T4_BAR2_QTYPE_INGRESS,
+ cq-bar2_qid,
+ user ? cq-bar2_pa : NULL);
+ if (user !cq-bar2_va) {
+ pr_warn(MOD %s: cqid %u not in BAR2 range.\n,
+ pci_name(rdev-lldi.pdev), cq-cqid);
+ ret = -EINVAL;
+ goto err4;
}
return 0;
err4:
@@ -971,7 +969,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
int entries,
insert_mmap(ucontext, mm);
mm2-key = uresp.gts_key;
- mm2-addr = chp-cq.ugts;
+ mm2-addr = (u64)(uintptr_t)chp-cq.bar2_pa;
Why are you using a cast here at all? bar2_pa is already u64...
So it should just have the (uintptr_t) cast?
No, it should be no cast at all. The uintptr_t cast is only for casting
an int-ptr or ptr-int. In those cases, if the size of an int != size
of ptr, you can loose data, and uintptr_t tells the compiler I know I'm
casting between possibly lossy data sizes and either A) I've checked and
it's OK or B) I'm ok with ptr truncation and the loss won't hurt us.
It basically turns off size checks when sticking a ptr into an int. You
should therefore use it only in those circumstances. For example, when
storing a cookie that doesn't have a strict uniqueness requirement, the
loss due to truncation is probably OK. Or if you know you are only
doing something like initially storing an int into a pointer, and then
later storing that pointer back into an int, so there can never be any
truncation because the source of the ptr was always int sized. Those
are the times to use uintptr. In this case, you have a real u64 going
into a real u64, there should be no casts.
--
Doug Ledford dledf...@redhat.com
GPG KeyID: 0E572FDD
signature.asc
Description: This is a digitally signed message part
Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities exceeding the page size
On Tue, 2015-06-09 at 09:18 -0500, Steve Wise wrote: Why are you using a cast here at all? bar2_pa is already u64... So it should just have the (uintptr_t) cast? No, it should be no cast at all. The uintptr_t cast is only for casting an int-ptr or ptr-int. In those cases, if the size of an int != size of ptr, you can loose data, and uintptr_t tells the compiler I know I'm casting between possibly lossy data sizes and either A) I've checked and it's OK or B) I'm ok with ptr truncation and the loss won't hurt us. It basically turns off size checks when sticking a ptr into an int. You should therefore use it only in those circumstances. For example, when storing a cookie that doesn't have a strict uniqueness requirement, the loss due to truncation is probably OK. Or if you know you are only doing something like initially storing an int into a pointer, and then later storing that pointer back into an int, so there can never be any truncation because the source of the ptr was always int sized. Those are the times to use uintptr. In this case, you have a real u64 going into a real u64, there should be no casts. My bad. I thought bar2_pa was a ptr... I didn't look up the actual structure definition, but: + cq-bar2_va = c4iw_bar2_addrs(rdev, cq-cqid, T4_BAR2_QTYPE_INGRESS, + cq-bar2_qid, + user ? cq-bar2_pa : NULL); +void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid, + enum cxgb4_bar2_qtype qtype, + unsigned int *pbar2_qid, u64 *pbar2_pa) Looks like either it's a u64 or else there should be compiler warnings about passing cq-bar2_pa to c4iw_bar2_addrs. -- Doug Ledford dledf...@redhat.com GPG KeyID: 0E572FDD signature.asc Description: This is a digitally signed message part
RE: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities exceeding the page size
-Original Message-
From: Doug Ledford [mailto:dledf...@redhat.com]
Sent: Tuesday, June 09, 2015 9:16 AM
To: Steve Wise
Cc: 'Hariprasad Shenai'; linux-rdma@vger.kernel.org; lee...@chelsio.com;
nirran...@chelsio.com
Subject: Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities
exceeding the page size
On Tue, 2015-06-09 at 09:07 -0500, Steve Wise wrote:
-Original Message-
From: Doug Ledford [mailto:dledf...@redhat.com]
Sent: Tuesday, June 09, 2015 9:03 AM
To: Hariprasad Shenai
Cc: linux-rdma@vger.kernel.org; sw...@opengridcomputing.com;
lee...@chelsio.com; nirran...@chelsio.com
Subject: Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities
exceeding the page size
On Tue, 2015-06-09 at 18:23 +0530, Hariprasad Shenai wrote:
Handle this configuration:
Queues Per Page * SGE BAR2 Queue Register Area Size Page Size
Use cxgb4_bar2_sge_qregs() to obtain the proper location within the
bar2 region for a given qid.
Rework the DB and GTS write functions to make use of this bar2 info.
Signed-off-by: Steve Wise sw...@opengridcomputing.com
Signed-off-by: Hariprasad Shenai haripra...@chelsio.com
---
drivers/infiniband/hw/cxgb4/cq.c | 22 ++--
drivers/infiniband/hw/cxgb4/device.c | 16 +++--
drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +--
drivers/infiniband/hw/cxgb4/qp.c | 64
++
drivers/infiniband/hw/cxgb4/t4.h | 60
---
5 files changed, 98 insertions(+), 69 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cq.c
b/drivers/infiniband/hw/cxgb4/cq.c
index 68ddb37..8e5bbcb 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -156,19 +156,17 @@ static int create_cq(struct c4iw_rdev *rdev,
struct t4_cq *cq,
goto err4;
cq-gen = 1;
+ cq-gts = rdev-lldi.gts_reg;
cq-rdev = rdev;
- if (user) {
- u32 off = (cq-cqid rdev-cqshift) PAGE_MASK;
- cq-ugts = (u64)rdev-bar2_pa + off;
- } else if (is_t4(rdev-lldi.adapter_type)) {
- cq-gts = rdev-lldi.gts_reg;
- cq-qid_mask = -1U;
- } else {
- u32 off = ((cq-cqid rdev-cqshift) PAGE_MASK) +
12;
-
- cq-gts = rdev-bar2_kva + off;
- cq-qid_mask = rdev-qpmask;
+ cq-bar2_va = c4iw_bar2_addrs(rdev, cq-cqid,
T4_BAR2_QTYPE_INGRESS,
+ cq-bar2_qid,
+ user ? cq-bar2_pa : NULL);
+ if (user !cq-bar2_va) {
+ pr_warn(MOD %s: cqid %u not in BAR2 range.\n,
+ pci_name(rdev-lldi.pdev), cq-cqid);
+ ret = -EINVAL;
+ goto err4;
}
return 0;
err4:
@@ -971,7 +969,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device
*ibdev, int entries,
insert_mmap(ucontext, mm);
mm2-key = uresp.gts_key;
- mm2-addr = chp-cq.ugts;
+ mm2-addr = (u64)(uintptr_t)chp-cq.bar2_pa;
Why are you using a cast here at all? bar2_pa is already u64...
So it should just have the (uintptr_t) cast?
No, it should be no cast at all. The uintptr_t cast is only for casting
an int-ptr or ptr-int. In those cases, if the size of an int != size
of ptr, you can loose data, and uintptr_t tells the compiler I know I'm
casting between possibly lossy data sizes and either A) I've checked and
it's OK or B) I'm ok with ptr truncation and the loss won't hurt us.
It basically turns off size checks when sticking a ptr into an int. You
should therefore use it only in those circumstances. For example, when
storing a cookie that doesn't have a strict uniqueness requirement, the
loss due to truncation is probably OK. Or if you know you are only
doing something like initially storing an int into a pointer, and then
later storing that pointer back into an int, so there can never be any
truncation because the source of the ptr was always int sized. Those
are the times to use uintptr. In this case, you have a real u64 going
into a real u64, there should be no casts.
My bad. I thought bar2_pa was a ptr...
--
To unsubscribe from this list: send the line unsubscribe linux-rdma in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH for-4.1 2/2] iw_cxgb4: support for bar2 qid densities exceeding the page size
On Tue, 2015-06-09 at 18:23 +0530, Hariprasad Shenai wrote:
Handle this configuration:
Queues Per Page * SGE BAR2 Queue Register Area Size Page Size
Use cxgb4_bar2_sge_qregs() to obtain the proper location within the
bar2 region for a given qid.
Rework the DB and GTS write functions to make use of this bar2 info.
Signed-off-by: Steve Wise sw...@opengridcomputing.com
Signed-off-by: Hariprasad Shenai haripra...@chelsio.com
---
drivers/infiniband/hw/cxgb4/cq.c | 22 ++--
drivers/infiniband/hw/cxgb4/device.c | 16 +++--
drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +--
drivers/infiniband/hw/cxgb4/qp.c | 64
++
drivers/infiniband/hw/cxgb4/t4.h | 60 ---
5 files changed, 98 insertions(+), 69 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cq.c
b/drivers/infiniband/hw/cxgb4/cq.c
index 68ddb37..8e5bbcb 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -156,19 +156,17 @@ static int create_cq(struct c4iw_rdev *rdev, struct
t4_cq *cq,
goto err4;
cq-gen = 1;
+ cq-gts = rdev-lldi.gts_reg;
cq-rdev = rdev;
- if (user) {
- u32 off = (cq-cqid rdev-cqshift) PAGE_MASK;
- cq-ugts = (u64)rdev-bar2_pa + off;
- } else if (is_t4(rdev-lldi.adapter_type)) {
- cq-gts = rdev-lldi.gts_reg;
- cq-qid_mask = -1U;
- } else {
- u32 off = ((cq-cqid rdev-cqshift) PAGE_MASK) + 12;
-
- cq-gts = rdev-bar2_kva + off;
- cq-qid_mask = rdev-qpmask;
+ cq-bar2_va = c4iw_bar2_addrs(rdev, cq-cqid, T4_BAR2_QTYPE_INGRESS,
+ cq-bar2_qid,
+ user ? cq-bar2_pa : NULL);
+ if (user !cq-bar2_va) {
+ pr_warn(MOD %s: cqid %u not in BAR2 range.\n,
+ pci_name(rdev-lldi.pdev), cq-cqid);
+ ret = -EINVAL;
+ goto err4;
}
return 0;
err4:
@@ -971,7 +969,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int
entries,
insert_mmap(ucontext, mm);
mm2-key = uresp.gts_key;
- mm2-addr = chp-cq.ugts;
+ mm2-addr = (u64)(uintptr_t)chp-cq.bar2_pa;
Why are you using a cast here at all? bar2_pa is already u64...
--
Doug Ledford dledf...@redhat.com
GPG KeyID: 0E572FDD
signature.asc
Description: This is a digitally signed message part
