Re: [PATCH 3.2] IB/qib: Change lkey table allocation to support more MRs
On Tue, 2015-09-29 at 10:51 -0400, Mike Marciniszyn wrote: > The lkey table is allocated with with a get_user_pages() with an > order based on a number of index bits from a module parameter. > > The underlying kernel code cannot allocate that many contiguous > pages. > > There is no reason the underlying memory needs to be physically > contiguous. > > This patch: > - switches the allocation/deallocation to vmalloc/vfree > - caps the number of bits to 23 to insure at least 1 generation bit > o this matches the module parameter description > > Cc: sta...@vger.kernel.org # 3.2 > Reviewed-by: Vinit Agnihotri > Signed-off-by: Mike Marciniszyn > Signed-off-by: Doug Ledford [...] qib_dev_warn() is not defined. But, I worked out how to backport this anyway. Ben. -- Ben Hutchings Once a job is fouled up, anything done to improve it makes it worse. signature.asc Description: This is a digitally signed message part
Re: [PATCH 3.2] IB/qib: Change lkey table allocation to support more MRs
On 02/10/2015 16:10, Marciniszyn, Mike wrote: >>> The lkey table is allocated with with a get_user_pages() with an >> Don't you mean __get_free_pages? >> > > I was a nit in the original upstream commit. > > I don’t think it is a big deal since the patch context clarifies. I agree it's not a big deal. I guess it only caught my attention since reading it made me wonder why you need pinned user memory for the lkey table. Haggai -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
RE: [PATCH 3.2] IB/qib: Change lkey table allocation to support more MRs
> > The lkey table is allocated with with a get_user_pages() with an
> Don't you mean __get_free_pages?
>
I was a nit in the original upstream commit.
I don’t think it is a big deal since the patch context clarifies.
Mike
N�r��yb�X��ǧv�^�){.n�+{��ٚ�{ay� ʇڙ�,j��f���h���z� �w���
���j:+v���w�j�mzZ+�ݢj"��!�i
Re: [PATCH 3.2] IB/qib: Change lkey table allocation to support more MRs
On 29/09/2015 17:51, Mike Marciniszyn wrote:
> The lkey table is allocated with with a get_user_pages() with an
Don't you mean __get_free_pages?
Regards,
Haggai
> order based on a number of index bits from a module parameter.
>
> The underlying kernel code cannot allocate that many contiguous pages.
>
> There is no reason the underlying memory needs to be physically
> contiguous.
>
> This patch:
> - switches the allocation/deallocation to vmalloc/vfree
> - caps the number of bits to 23 to insure at least 1 generation bit
> o this matches the module parameter description
>
> Cc: sta...@vger.kernel.org # 3.2
> Reviewed-by: Vinit Agnihotri
> Signed-off-by: Mike Marciniszyn
> Signed-off-by: Doug Ledford
> ---
> drivers/infiniband/hw/qib/qib_keys.c |4
> drivers/infiniband/hw/qib/qib_verbs.c | 16 +++-
> drivers/infiniband/hw/qib/qib_verbs.h |2 ++
> 3 files changed, 17 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/infiniband/hw/qib/qib_keys.c
> b/drivers/infiniband/hw/qib/qib_keys.c
> index 8fd19a4..ca6e6cf 100644
> --- a/drivers/infiniband/hw/qib/qib_keys.c
> +++ b/drivers/infiniband/hw/qib/qib_keys.c
> @@ -69,6 +69,10 @@ int qib_alloc_lkey(struct qib_lkey_table *rkt, struct
> qib_mregion *mr)
>* unrestricted LKEY.
>*/
> rkt->gen++;
> + /*
> + * bits are capped in qib_verbs.c to insure enough bits
> + * for generation number
> + */
> mr->lkey = (r << (32 - ib_qib_lkey_table_size)) |
> 1 << (24 - ib_qib_lkey_table_size)) - 1) & rkt->gen)
><< 8);
> diff --git a/drivers/infiniband/hw/qib/qib_verbs.c
> b/drivers/infiniband/hw/qib/qib_verbs.c
> index a894762..95df4cd 100644
> --- a/drivers/infiniband/hw/qib/qib_verbs.c
> +++ b/drivers/infiniband/hw/qib/qib_verbs.c
> @@ -40,6 +40,7 @@
> #include
> #include
> #include
> +#include
>
> #include "qib.h"
> #include "qib_common.h"
> @@ -2035,10 +2036,16 @@ int qib_register_ib_device(struct qib_devdata *dd)
>* the LKEY). The remaining bits act as a generation number or tag.
>*/
> spin_lock_init(&dev->lk_table.lock);
> + /* insure generation is at least 4 bits see keys.c */
> + if (ib_qib_lkey_table_size > MAX_LKEY_TABLE_BITS) {
> + qib_dev_warn(dd, "lkey bits %u too large, reduced to %u\n",
> + ib_qib_lkey_table_size, MAX_LKEY_TABLE_BITS);
> + ib_qib_lkey_table_size = MAX_LKEY_TABLE_BITS;
> + }
> dev->lk_table.max = 1 << ib_qib_lkey_table_size;
> lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
> - dev->lk_table.table = (struct qib_mregion **)
> - __get_free_pages(GFP_KERNEL, get_order(lk_tab_size));
> + dev->lk_table.table = (struct qib_mregion __rcu **)
> + vmalloc(lk_tab_size);
> if (dev->lk_table.table == NULL) {
> ret = -ENOMEM;
> goto err_lk;
> @@ -2208,7 +2215,7 @@ err_tx:
> sizeof(struct qib_pio_header),
> dev->pio_hdrs, dev->pio_hdrs_phys);
> err_hdrs:
> - free_pages((unsigned long) dev->lk_table.table, get_order(lk_tab_size));
> + vfree(dev->lk_table.table);
> err_lk:
> kfree(dev->qp_table);
> err_qpt:
> @@ -2262,7 +2269,6 @@ void qib_unregister_ib_device(struct qib_devdata *dd)
> sizeof(struct qib_pio_header),
> dev->pio_hdrs, dev->pio_hdrs_phys);
> lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
> - free_pages((unsigned long) dev->lk_table.table,
> -get_order(lk_tab_size));
> + vfree(dev->lk_table.table);
> kfree(dev->qp_table);
> }
> diff --git a/drivers/infiniband/hw/qib/qib_verbs.h
> b/drivers/infiniband/hw/qib/qib_verbs.h
> index 0c19ef0..66f7f62 100644
> --- a/drivers/infiniband/hw/qib/qib_verbs.h
> +++ b/drivers/infiniband/hw/qib/qib_verbs.h
> @@ -622,6 +622,8 @@ struct qib_qpn_table {
> struct qpn_map map[QPNMAP_ENTRIES];
> };
>
> +#define MAX_LKEY_TABLE_BITS 23
> +
> struct qib_lkey_table {
> spinlock_t lock; /* protect changes in this struct */
> u32 next; /* next unused index (speeds search) */
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 3.2] IB/qib: Change lkey table allocation to support more MRs
The lkey table is allocated with with a get_user_pages() with an
order based on a number of index bits from a module parameter.
The underlying kernel code cannot allocate that many contiguous pages.
There is no reason the underlying memory needs to be physically
contiguous.
This patch:
- switches the allocation/deallocation to vmalloc/vfree
- caps the number of bits to 23 to insure at least 1 generation bit
o this matches the module parameter description
Cc: sta...@vger.kernel.org # 3.2
Reviewed-by: Vinit Agnihotri
Signed-off-by: Mike Marciniszyn
Signed-off-by: Doug Ledford
---
drivers/infiniband/hw/qib/qib_keys.c |4
drivers/infiniband/hw/qib/qib_verbs.c | 16 +++-
drivers/infiniband/hw/qib/qib_verbs.h |2 ++
3 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/drivers/infiniband/hw/qib/qib_keys.c
b/drivers/infiniband/hw/qib/qib_keys.c
index 8fd19a4..ca6e6cf 100644
--- a/drivers/infiniband/hw/qib/qib_keys.c
+++ b/drivers/infiniband/hw/qib/qib_keys.c
@@ -69,6 +69,10 @@ int qib_alloc_lkey(struct qib_lkey_table *rkt, struct
qib_mregion *mr)
* unrestricted LKEY.
*/
rkt->gen++;
+ /*
+* bits are capped in qib_verbs.c to insure enough bits
+* for generation number
+*/
mr->lkey = (r << (32 - ib_qib_lkey_table_size)) |
1 << (24 - ib_qib_lkey_table_size)) - 1) & rkt->gen)
<< 8);
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c
b/drivers/infiniband/hw/qib/qib_verbs.c
index a894762..95df4cd 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -40,6 +40,7 @@
#include
#include
#include
+#include
#include "qib.h"
#include "qib_common.h"
@@ -2035,10 +2036,16 @@ int qib_register_ib_device(struct qib_devdata *dd)
* the LKEY). The remaining bits act as a generation number or tag.
*/
spin_lock_init(&dev->lk_table.lock);
+ /* insure generation is at least 4 bits see keys.c */
+ if (ib_qib_lkey_table_size > MAX_LKEY_TABLE_BITS) {
+ qib_dev_warn(dd, "lkey bits %u too large, reduced to %u\n",
+ ib_qib_lkey_table_size, MAX_LKEY_TABLE_BITS);
+ ib_qib_lkey_table_size = MAX_LKEY_TABLE_BITS;
+ }
dev->lk_table.max = 1 << ib_qib_lkey_table_size;
lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
- dev->lk_table.table = (struct qib_mregion **)
- __get_free_pages(GFP_KERNEL, get_order(lk_tab_size));
+ dev->lk_table.table = (struct qib_mregion __rcu **)
+ vmalloc(lk_tab_size);
if (dev->lk_table.table == NULL) {
ret = -ENOMEM;
goto err_lk;
@@ -2208,7 +2215,7 @@ err_tx:
sizeof(struct qib_pio_header),
dev->pio_hdrs, dev->pio_hdrs_phys);
err_hdrs:
- free_pages((unsigned long) dev->lk_table.table, get_order(lk_tab_size));
+ vfree(dev->lk_table.table);
err_lk:
kfree(dev->qp_table);
err_qpt:
@@ -2262,7 +2269,6 @@ void qib_unregister_ib_device(struct qib_devdata *dd)
sizeof(struct qib_pio_header),
dev->pio_hdrs, dev->pio_hdrs_phys);
lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
- free_pages((unsigned long) dev->lk_table.table,
- get_order(lk_tab_size));
+ vfree(dev->lk_table.table);
kfree(dev->qp_table);
}
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h
b/drivers/infiniband/hw/qib/qib_verbs.h
index 0c19ef0..66f7f62 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -622,6 +622,8 @@ struct qib_qpn_table {
struct qpn_map map[QPNMAP_ENTRIES];
};
+#define MAX_LKEY_TABLE_BITS 23
+
struct qib_lkey_table {
spinlock_t lock; /* protect changes in this struct */
u32 next; /* next unused index (speeds search) */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
