RE: [PATCH V6 8/9] isert: Use local_dma_lkey whenever possible.

2015-07-24 Thread Steve Wise 


> -Original Message-
> From: Jason Gunthorpe [mailto:jguntho...@obsidianresearch.com]
> Sent: Friday, July 24, 2015 11:49 AM
> To: Steve Wise
> Cc: dledf...@redhat.com; infinip...@intel.com; sa...@mellanox.com; 
> ogerl...@mellanox.com; r...@mellanox.com; linux-
> r...@vger.kernel.org; e...@mellanox.com; target-de...@vger.kernel.org; 
> linux-...@vger.kernel.org; bfie...@fieldses.org
> Subject: Re: [PATCH V6 8/9] isert: Use local_dma_lkey whenever possible.
> 
> On Fri, Jul 24, 2015 at 11:18:59AM -0500, Steve Wise wrote:
> > No need to allocate a dma_mr if the device provides a local_dma_lkey.
> 
> It is probably safe to put your series on top of mine, which
> incorporates this patch already.
> 
> https://github.com/jgunthorpe/linux/commits/remove-ib_get_dma_mr
> 
> Jason

I will rebase on this. 

 

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH V6 8/9] isert: Use local_dma_lkey whenever possible.

2015-07-24 Thread Jason Gunthorpe 
On Fri, Jul 24, 2015 at 11:18:59AM -0500, Steve Wise wrote:
> No need to allocate a dma_mr if the device provides a local_dma_lkey.

It is probably safe to put your series on top of mine, which
incorporates this patch already.

https://github.com/jgunthorpe/linux/commits/remove-ib_get_dma_mr

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH V6 8/9] isert: Use local_dma_lkey whenever possible.

2015-07-24 Thread Steve Wise 
No need to allocate a dma_mr if the device provides a local_dma_lkey.

Signed-off-by: Steve Wise 
---

 drivers/infiniband/ulp/isert/ib_isert.c |   47 ++-
 drivers/infiniband/ulp/isert/ib_isert.h |1 +
 2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/drivers/infiniband/ulp/isert/ib_isert.c 
b/drivers/infiniband/ulp/isert/ib_isert.c
index 8ae9208..47bb790 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -249,7 +249,7 @@ isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
rx_sg = &rx_desc->rx_sg;
rx_sg->addr = rx_desc->dma_addr;
rx_sg->length = ISER_RX_PAYLOAD_SIZE;
-   rx_sg->lkey = device->mr->lkey;
+   rx_sg->lkey = device->local_dma_lkey;
}
 
isert_conn->rx_desc_head = 0;
@@ -375,8 +375,9 @@ isert_create_device_ib_res(struct isert_device *device)
if (ret)
return ret;
 
-   /* asign function handlers */
+   /* assign function handlers */
if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS &&
+   dev_attr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY &&
dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) {
device->use_fastreg = 1;
device->reg_rdma_mem = isert_reg_frmr_pi;
@@ -399,12 +400,17 @@ isert_create_device_ib_res(struct isert_device *device)
goto out_cq;
}
 
-   device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE);
-   if (IS_ERR(device->mr)) {
-   ret = PTR_ERR(device->mr);
-   isert_err("failed to create dma mr, device %p, ret=%d\n",
- device, ret);
-   goto out_mr;
+   if (device->use_fastreg)
+   device->local_dma_lkey = device->ib_device->local_dma_lkey;
+   else {
+   device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE);
+   if (IS_ERR(device->mr)) {
+   ret = PTR_ERR(device->mr);
+   isert_err("failed to create dma mr, device %p, 
ret=%d\n",
+ device, ret);
+   goto out_mr;
+   }
+   device->local_dma_lkey = device->mr->lkey;
}
 
/* Check signature cap */
@@ -425,7 +431,8 @@ isert_free_device_ib_res(struct isert_device *device)
 {
isert_info("device %p\n", device);
 
-   ib_dereg_mr(device->mr);
+   if (!device->use_fastreg)
+   ib_dereg_mr(device->mr);
ib_dealloc_pd(device->pd);
isert_free_comps(device);
 }
@@ -1108,8 +1115,8 @@ isert_create_send_desc(struct isert_conn *isert_conn,
tx_desc->num_sge = 1;
tx_desc->isert_cmd = isert_cmd;
 
-   if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
-   tx_desc->tx_sg[0].lkey = device->mr->lkey;
+   if (tx_desc->tx_sg[0].lkey != device->local_dma_lkey) {
+   tx_desc->tx_sg[0].lkey = device->local_dma_lkey;
isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
}
 }
@@ -1132,7 +1139,7 @@ isert_init_tx_hdrs(struct isert_conn *isert_conn,
tx_desc->dma_addr = dma_addr;
tx_desc->tx_sg[0].addr  = tx_desc->dma_addr;
tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
-   tx_desc->tx_sg[0].lkey = device->mr->lkey;
+   tx_desc->tx_sg[0].lkey = device->local_dma_lkey;
 
isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
  tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
@@ -1165,7 +1172,7 @@ isert_rdma_post_recvl(struct isert_conn *isert_conn)
memset(&sge, 0, sizeof(struct ib_sge));
sge.addr = isert_conn->login_req_dma;
sge.length = ISER_RX_LOGIN_SIZE;
-   sge.lkey = isert_conn->device->mr->lkey;
+   sge.lkey = isert_conn->device->local_dma_lkey;
 
isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
sge.addr, sge.length, sge.lkey);
@@ -1215,7 +1222,7 @@ isert_put_login_tx(struct iscsi_conn *conn, struct 
iscsi_login *login,
 
tx_dsg->addr= isert_conn->login_rsp_dma;
tx_dsg->length  = length;
-   tx_dsg->lkey= isert_conn->device->mr->lkey;
+   tx_dsg->lkey= isert_conn->device->local_dma_lkey;
tx_desc->num_sge = 2;
}
if (!login->login_failed) {
@@ -1734,6 +1741,7 @@ isert_unmap_lkey(struct isert_cmd *isert_cmd, struct 
isert_conn *isert_conn)
 
if (wr->send_wr) {
isert_dbg("Cmd %p free send_wr\n", isert_cmd);
+   wr->send_wr_num = 0;
kfree(wr->send_wr);
wr->send_wr = NULL;
}
@@ -1967,7 +1975,6 @@ isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
iscsit_stop_dataout_timer(cmd);
device->unreg_rdma_mem(isert_cmd, isert_conn);
cmd->write_data_done =