Re: [PATCH 3/6] qla2xxx: Add FC-NVMe F/W initialization and transport registration

[Madhani, Himanshu]

> On Jun 19, 2017, at 2:22 PM, James Smart  wrote:
> 
> On 6/16/2017 3:47 PM, Himanshu Madhani wrote:
>> From: Duane Grigsby 
>> 
>> This code provides the interfaces to register remote and local ports
>> of FC4 type 0x28 with the FC-NVMe transport and transports the
>> requests (FC-NVMe FC link services and FC-NVMe commands IUs) to the
>> fabric. It also provides the support for allocating h/w queues and
>> aborting FC-NVMe FC requests.
>> 
>> Signed-off-by: Darren Trapp 
>> Signed-off-by: Duane Grigsby 
>> Signed-off-by: Anil Gurumurthy 
>> Signed-off-by: Giridhar Malavali 
>> Signed-off-by: Himanshu Madhani 
>> 
>> +#if (IS_ENABLED(CONFIG_NVME_FC))
>> +extern int nvme_fc_register_remoteport(struct nvme_fc_local_port *,
>> +struct nvme_fc_port_info *, struct nvme_fc_remote_port **);
>> +extern int nvme_fc_register_localport(struct nvme_fc_port_info *,
>> +struct nvme_fc_port_template *, struct device *,
>> +struct nvme_fc_local_port **);
>> +extern int nvme_fc_unregister_localport(struct nvme_fc_local_port *);
>> +#endif
> 
> There should be no need for these. They are in the 
> include/linux/nvme-fc-driver.h header.
> 

Agree. Will remove it 

> 
>> +static void qla_nvme_sp_ls_done(void *ptr, int res)
>> +{
>> +srb_t *sp = ptr;
>> +struct srb_iocb *nvme;
>> +struct nvmefc_ls_req   *fd;
>> +struct nvme_private *priv;
>> +
>> +if (atomic_read(>ref_count) == 0) {
>> +ql_log(ql_log_warn, sp->fcport->vha, 0x2123,
>> +"SP reference-count to ZERO on LS_done -- sp=%p.\n", sp);
>> +return;
>> +}
>> +
>> +if (!atomic_dec_and_test(>ref_count))
>> +return;
>> +
>> +if (res)
>> +res = NVME_SC_FC_TRANSPORT_ERROR;
> 
> LS request failures status should be -Exxx values, not nvme status codes 
> (which don't apply to LS's).
> The api probably wasn't super clear on this.
> 

Will fix up in v2.

>> +
>> +nvme = >u.iocb_cmd;
>> +fd = nvme->u.nvme.desc;
>> +priv = fd->private;
>> +priv->comp_status = res;
>> +schedule_work(>ls_work);
>> +/* work schedule doesn't need the sp */
>> +qla2x00_rel_sp(sp);
>> +}
>> +
>> +static void qla_nvme_sp_done(void *ptr, int res)
>> +{
>> +srb_t *sp = ptr;
>> +struct srb_iocb *nvme;
>> +struct nvmefc_fcp_req *fd;
>> +
>> +nvme = >u.iocb_cmd;
>> +fd = nvme->u.nvme.desc;
>> +
>> +if (!atomic_dec_and_test(>ref_count))
>> +return;
>> +
>> +if (!(sp->fcport->nvme_flag & NVME_FLAG_REGISTERED))
>> +goto rel;
>> +
>> +if (unlikely((nvme->u.nvme.comp_status) || res))
>> +fd->status = NVME_SC_FC_TRANSPORT_ABORTED;
>> +else
>> +fd->status = 0;
> 
> FCP req failures status should be -Exxx values, not nvme status codes. This 
> is clear in the api header.
> 

Sure. Will fix this in v2. 

> -- james
> 

Re: [PATCH 3/6] qla2xxx: Add FC-NVMe F/W initialization and transport registration

[James Smart]

On 6/16/2017 3:47 PM, Himanshu Madhani wrote:

From: Duane Grigsby 

This code provides the interfaces to register remote and local ports
of FC4 type 0x28 with the FC-NVMe transport and transports the
requests (FC-NVMe FC link services and FC-NVMe commands IUs) to the
fabric. It also provides the support for allocating h/w queues and
aborting FC-NVMe FC requests.

Signed-off-by: Darren Trapp 
Signed-off-by: Duane Grigsby 
Signed-off-by: Anil Gurumurthy 
Signed-off-by: Giridhar Malavali 
Signed-off-by: Himanshu Madhani 

+#if (IS_ENABLED(CONFIG_NVME_FC))
+extern int nvme_fc_register_remoteport(struct nvme_fc_local_port *,
+struct nvme_fc_port_info *, struct nvme_fc_remote_port **);
+extern int nvme_fc_register_localport(struct nvme_fc_port_info *,
+struct nvme_fc_port_template *, struct device *,
+struct nvme_fc_local_port **);
+extern int nvme_fc_unregister_localport(struct nvme_fc_local_port *);
+#endif


There should be no need for these. They are in the 
include/linux/nvme-fc-driver.h header.




+static void qla_nvme_sp_ls_done(void *ptr, int res)
+{
+   srb_t *sp = ptr;
+   struct srb_iocb *nvme;
+   struct nvmefc_ls_req   *fd;
+   struct nvme_private *priv;
+
+   if (atomic_read(>ref_count) == 0) {
+   ql_log(ql_log_warn, sp->fcport->vha, 0x2123,
+   "SP reference-count to ZERO on LS_done -- sp=%p.\n", sp);
+   return;
+   }
+
+   if (!atomic_dec_and_test(>ref_count))
+   return;
+
+   if (res)
+   res = NVME_SC_FC_TRANSPORT_ERROR;


LS request failures status should be -Exxx values, not nvme status codes 
(which don't apply to LS's).

The api probably wasn't super clear on this.


+
+   nvme = >u.iocb_cmd;
+   fd = nvme->u.nvme.desc;
+   priv = fd->private;
+   priv->comp_status = res;
+   schedule_work(>ls_work);
+   /* work schedule doesn't need the sp */
+   qla2x00_rel_sp(sp);
+}
+
+static void qla_nvme_sp_done(void *ptr, int res)
+{
+   srb_t *sp = ptr;
+   struct srb_iocb *nvme;
+   struct nvmefc_fcp_req *fd;
+
+   nvme = >u.iocb_cmd;
+   fd = nvme->u.nvme.desc;
+
+   if (!atomic_dec_and_test(>ref_count))
+   return;
+
+   if (!(sp->fcport->nvme_flag & NVME_FLAG_REGISTERED))
+   goto rel;
+
+   if (unlikely((nvme->u.nvme.comp_status) || res))
+   fd->status = NVME_SC_FC_TRANSPORT_ABORTED;
+   else
+   fd->status = 0;


FCP req failures status should be -Exxx values, not nvme status codes. 
This is clear in the api header.


-- james

Re: [PATCH 3/6] qla2xxx: Add FC-NVMe F/W initialization and transport registration

[Madhani, Himanshu]

> On Jun 19, 2017, at 3:01 AM, Johannes Thumshirn  wrote:
> 
> On Fri, Jun 16, 2017 at 03:47:41PM -0700, Himanshu Madhani wrote:
> [...]
>> /*
>> + * Global functions prototype in qla_nvme.c source file.
>> + */
>> +extern void qla_nvme_register_hba(scsi_qla_host_t *);
>> +extern int  qla_nvme_register_remote(scsi_qla_host_t *, fc_port_t *);
>> +extern void qla_nvme_delete(scsi_qla_host_t *);
>> +extern void qla_nvme_abort(struct qla_hw_data *, srb_t *sp);
>> +extern void qla_nvme_unregister_remote_port(struct work_struct *);
>> +extern int qla_nvme_wait_on_rport_del(fc_port_t *);
>> +extern void qla_nvme_abort_all(fc_port_t *);
>> +extern int qla_nvme_post_cmd(struct nvme_fc_local_port *,
>> +struct nvme_fc_remote_port *, void *, struct nvmefc_fcp_req *);
>> +extern int qla_nvme_alloc_queue(struct nvme_fc_local_port *, unsigned int,
>> +u16, void **);
>> +extern int qla_nvme_hba_scan(scsi_qla_host_t *);
>> +extern void qla_nvme_ls_abort(struct nvme_fc_local_port *,
>> +struct nvme_fc_remote_port *, struct nvmefc_ls_req *);
>> +extern int qla_nvme_ls_req(struct nvme_fc_local_port *,
>> +struct nvme_fc_remote_port *, struct nvmefc_ls_req *);
>> +extern void qla_nvme_poll(struct nvme_fc_local_port *, void *);
>> +extern int qla2x00_start_nvme_mq(srb_t *);
>> +extern void qla_nvme_fcp_abort(struct nvme_fc_local_port *,
>> +struct nvme_fc_remote_port *, void *, struct nvmefc_fcp_req *);
>> +extern void qla24xx_nvme_ls4_iocb(scsi_qla_host_t *, struct pt_ls4_request 
>> *,
>> +struct req_que *);
> 
> Can't these go into a header instead of using extern?
> 

Sure.

> [...]
> 
>> @@ -4662,6 +4667,9 @@ qla2x00_configure_fabric(scsi_qla_host_t *vha)
>>  break;
>>  } while (0);
>> 
>> +if ((!vha->nvme_local_port) && (vha->flags.nvme_enabled))
> 
> 
> [...]
> 
>> +if ((ql2xnvmeenable) && IS_QLA27XX(ha))
>> +mcp->mb[4] |= NVME_ENABLE_FLAG;
>> +
> 
> [...]
> 
>> +
>> +/* bit 26 of fw_attributes */
>> +if ((ha->fw_attributes_h & 0x400) && (ql2xnvmeenable)) {
> 
> Superfluous parenthesis.
> 

Good catch. will fix in v2

> [...]
> 
>> +vha->flags.nvme_enabled = 1;
>> +icb->firmware_options_2 &=
>> +cpu_to_le32(~(BIT_0 | BIT_1 | BIT_2 | BIT_3));
> 
> 0xfff0 or maybe even ~0xf?
> 

will fix in v2

> [...]
> 
>> +ret = nvme_fc_register_remoteport(vha->nvme_local_port, >req,
>> +>nvme_remote_port);
>> +if (ret) {
>> +ql_log(ql_log_warn, vha, 0x212e,
>> +"Failed to register remote port. Transport returned %d\n",
>> +ret);
>> +return ret;
>> +}
>> +
>> +fcport->nvme_remote_port->private = fcport;
> 
> A FC-NVMe remote port can be in used right after a call to
> nvme_fc_register_remoteport(), if I'm not mistaken. So you should add the
> fcport to the nvme_remote_port->private _before_ calling
> nvme_fc_register_remoteport().

Will check this out. 

> 
>> +fcport->nvme_flag |= NVME_FLAG_REGISTERED;
>> +atomic_set(>nvme_ref_count, 1);
>> +rport->fcport = fcport;
>> +list_add_tail(>list, >nvme_rport_list);
>> +#endif
>> +return 0;
>> +}
>> +
>> +/*
>> + * Perform a scan and register those ports with FC-NVMe transport
>> + * input:  HBA handle
>> + */
>> +int qla_nvme_hba_scan(scsi_qla_host_t *vha)
> 
> This should be bool.
> 
>> +{
>> +#if (IS_ENABLED(CONFIG_NVME_FC))
>> +fc_port_t   *fcport;
>> +uint8_t not_found = 1;
> 
> bool found = false;
> 
>> +
>> +/*
>> + * We are using the first HBA found
>> + * Find matching fcport
>> + */
>> +list_for_each_entry(fcport, >vp_fcports, list) {
>> +if ((fcport->fc4f_nvme) &&
>> +(!(fcport->nvme_flag & NVME_FLAG_REGISTERED))) {
>> +qla_nvme_register_remote(vha, fcport);
>> +not_found = 0;
> 
> found = true;
> 
>> +}
>> +}
>> +
>> +return not_found;
> 
> return found;
> 
>> +#else
>> +return 1;
> 
> return false;
> 
>> +#endif
>> +}
> 
> And update the call-sites.
> 

Ack. 

> [...]
> 
>> +if (unlikely((nvme->u.nvme.comp_status) || res))
> 
> Parenthesis again.
> 

Ack. 

> [...]
> 
>> +void qla_nvme_ls_abort(struct nvme_fc_local_port *lport,
>> +struct nvme_fc_remote_port *rport, struct nvmefc_ls_req *fd)
>> +{
>> +struct nvme_private *priv = fd->private;
>> +fc_port_t *fcport = rport->private;
>> +srb_t *sp = priv->sp;
>> +int rval;
>> +struct qla_hw_data *ha;
>> +
>> +ql_log(ql_log_info, fcport->vha, 0x212b,
>> +"%s: aborting sp:%p on fcport:%p\n", __func__, sp, fcport);
>> +
>> +ha = fcport->vha->hw;
>> +rval = ha->isp_ops->abort_command(sp);
>> +if (rval != QLA_SUCCESS)
>> +ql_log(ql_log_warn, fcport->vha, 0x2125,
>> +"%s: failed to abort LS command for SP:%p rval=%x\n",
>> +

Re: [PATCH 3/6] qla2xxx: Add FC-NVMe F/W initialization and transport registration

[Johannes Thumshirn]

On Fri, Jun 16, 2017 at 03:47:41PM -0700, Himanshu Madhani wrote:
[...]
>  /*
> + * Global functions prototype in qla_nvme.c source file.
> + */
> +extern void qla_nvme_register_hba(scsi_qla_host_t *);
> +extern int  qla_nvme_register_remote(scsi_qla_host_t *, fc_port_t *);
> +extern void qla_nvme_delete(scsi_qla_host_t *);
> +extern void qla_nvme_abort(struct qla_hw_data *, srb_t *sp);
> +extern void qla_nvme_unregister_remote_port(struct work_struct *);
> +extern int qla_nvme_wait_on_rport_del(fc_port_t *);
> +extern void qla_nvme_abort_all(fc_port_t *);
> +extern int qla_nvme_post_cmd(struct nvme_fc_local_port *,
> +struct nvme_fc_remote_port *, void *, struct nvmefc_fcp_req *);
> +extern int qla_nvme_alloc_queue(struct nvme_fc_local_port *, unsigned int,
> +u16, void **);
> +extern int qla_nvme_hba_scan(scsi_qla_host_t *);
> +extern void qla_nvme_ls_abort(struct nvme_fc_local_port *,
> +struct nvme_fc_remote_port *, struct nvmefc_ls_req *);
> +extern int qla_nvme_ls_req(struct nvme_fc_local_port *,
> +struct nvme_fc_remote_port *, struct nvmefc_ls_req *);
> +extern void qla_nvme_poll(struct nvme_fc_local_port *, void *);
> +extern int qla2x00_start_nvme_mq(srb_t *);
> +extern void qla_nvme_fcp_abort(struct nvme_fc_local_port *,
> +struct nvme_fc_remote_port *, void *, struct nvmefc_fcp_req *);
> +extern void qla24xx_nvme_ls4_iocb(scsi_qla_host_t *, struct pt_ls4_request *,
> +struct req_que *);

Can't these go into a header instead of using extern?

[...]

> @@ -4662,6 +4667,9 @@ qla2x00_configure_fabric(scsi_qla_host_t *vha)
>   break;
>   } while (0);
>  
> + if ((!vha->nvme_local_port) && (vha->flags.nvme_enabled))


[...]

> + if ((ql2xnvmeenable) && IS_QLA27XX(ha))
> + mcp->mb[4] |= NVME_ENABLE_FLAG;
> +

[...]

> +
> + /* bit 26 of fw_attributes */
> + if ((ha->fw_attributes_h & 0x400) && (ql2xnvmeenable)) {

Superfluous parenthesis.

[...]

> + vha->flags.nvme_enabled = 1;
> + icb->firmware_options_2 &=
> + cpu_to_le32(~(BIT_0 | BIT_1 | BIT_2 | BIT_3));

0xfff0 or maybe even ~0xf?

[...]

> + ret = nvme_fc_register_remoteport(vha->nvme_local_port, >req,
> + >nvme_remote_port);
> + if (ret) {
> + ql_log(ql_log_warn, vha, 0x212e,
> + "Failed to register remote port. Transport returned %d\n",
> + ret);
> + return ret;
> + }
> +
> + fcport->nvme_remote_port->private = fcport;

A FC-NVMe remote port can be in used right after a call to
nvme_fc_register_remoteport(), if I'm not mistaken. So you should add the
fcport to the nvme_remote_port->private _before_ calling
nvme_fc_register_remoteport().

> + fcport->nvme_flag |= NVME_FLAG_REGISTERED;
> + atomic_set(>nvme_ref_count, 1);
> + rport->fcport = fcport;
> + list_add_tail(>list, >nvme_rport_list);
> +#endif
> + return 0;
> +}
> +
> +/*
> + * Perform a scan and register those ports with FC-NVMe transport
> + * input:  HBA handle
> + */
> +int qla_nvme_hba_scan(scsi_qla_host_t *vha)

This should be bool.

> +{
> +#if (IS_ENABLED(CONFIG_NVME_FC))
> + fc_port_t   *fcport;
> + uint8_t not_found = 1;

bool found = false;

> +
> + /*
> +  * We are using the first HBA found
> +  * Find matching fcport
> +  */
> + list_for_each_entry(fcport, >vp_fcports, list) {
> + if ((fcport->fc4f_nvme) &&
> + (!(fcport->nvme_flag & NVME_FLAG_REGISTERED))) {
> + qla_nvme_register_remote(vha, fcport);
> + not_found = 0;

found = true;

> + }
> + }
> +
> + return not_found;

return found;

> +#else
> + return 1;

return false;

> +#endif
> +}

And update the call-sites.

[...]

> + if (unlikely((nvme->u.nvme.comp_status) || res))

Parenthesis again.

[...]

> +void qla_nvme_ls_abort(struct nvme_fc_local_port *lport,
> +struct nvme_fc_remote_port *rport, struct nvmefc_ls_req *fd)
> +{
> + struct nvme_private *priv = fd->private;
> + fc_port_t *fcport = rport->private;
> + srb_t *sp = priv->sp;
> + int rval;
> + struct qla_hw_data *ha;
> +
> + ql_log(ql_log_info, fcport->vha, 0x212b,
> + "%s: aborting sp:%p on fcport:%p\n", __func__, sp, fcport);
> +
> + ha = fcport->vha->hw;
> + rval = ha->isp_ops->abort_command(sp);
> + if (rval != QLA_SUCCESS)
> + ql_log(ql_log_warn, fcport->vha, 0x2125,
> + "%s: failed to abort LS command for SP:%p rval=%x\n",
> + __func__, sp, rval);
> +}

I think qla_nvme_ls_abort() is a bit too verbose, it nearly only consists of
logging and thus hides the call to abort_command() from the reader.

> +
> +static void qla_nvme_ls_complete(struct work_struct *work)
> +{
> + struct nvme_private *priv =
> + container_of(work, struct nvme_private, ls_work);
> 

[PATCH 3/6] qla2xxx: Add FC-NVMe F/W initialization and transport registration

[Himanshu Madhani]

From: Duane Grigsby 

This code provides the interfaces to register remote and local ports
of FC4 type 0x28 with the FC-NVMe transport and transports the
requests (FC-NVMe FC link services and FC-NVMe commands IUs) to the
fabric. It also provides the support for allocating h/w queues and
aborting FC-NVMe FC requests.

Signed-off-by: Darren Trapp 
Signed-off-by: Duane Grigsby 
Signed-off-by: Anil Gurumurthy 
Signed-off-by: Giridhar Malavali 
Signed-off-by: Himanshu Madhani 
---
 drivers/scsi/qla2xxx/Makefile   |   2 +-
 drivers/scsi/qla2xxx/qla_dbg.c  |   2 +-
 drivers/scsi/qla2xxx/qla_def.h  |   3 +
 drivers/scsi/qla2xxx/qla_gbl.h  |  27 ++
 drivers/scsi/qla2xxx/qla_init.c |   8 +
 drivers/scsi/qla2xxx/qla_iocb.c |  36 ++
 drivers/scsi/qla2xxx/qla_isr.c  |  19 +
 drivers/scsi/qla2xxx/qla_mbx.c  |  22 ++
 drivers/scsi/qla2xxx/qla_nvme.c | 797 
 drivers/scsi/qla2xxx/qla_nvme.h | 132 +++
 drivers/scsi/qla2xxx/qla_os.c   |  40 +-
 11 files changed, 1079 insertions(+), 9 deletions(-)
 create mode 100644 drivers/scsi/qla2xxx/qla_nvme.c
 create mode 100644 drivers/scsi/qla2xxx/qla_nvme.h

diff --git a/drivers/scsi/qla2xxx/Makefile b/drivers/scsi/qla2xxx/Makefile
index 44def6bb4bb0..0b767a0bb308 100644
--- a/drivers/scsi/qla2xxx/Makefile
+++ b/drivers/scsi/qla2xxx/Makefile
@@ -1,6 +1,6 @@
 qla2xxx-y := qla_os.o qla_init.o qla_mbx.o qla_iocb.o qla_isr.o qla_gs.o \
qla_dbg.o qla_sup.o qla_attr.o qla_mid.o qla_dfs.o qla_bsg.o \
-   qla_nx.o qla_mr.o qla_nx2.o qla_target.o qla_tmpl.o
+   qla_nx.o qla_mr.o qla_nx2.o qla_target.o qla_tmpl.o qla_nvme.o
 
 obj-$(CONFIG_SCSI_QLA_FC) += qla2xxx.o
 obj-$(CONFIG_TCM_QLA2XXX) += tcm_qla2xxx.o
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index cf4f47603a91..d840529fc023 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -15,7 +15,7 @@
  * |  || 0x015b-0x0160 |
  * |  || 0x016e
|
  * | Mailbox commands |   0x1199   | 0x1193
|
- * | Device Discovery |   0x2131   | 0x210e-0x2116  |
+ * | Device Discovery |   0x2134   | 0x210e-0x2116  |
  * | || 0x211a |
  * |  || 0x211c-0x2128  |
  * |  || 0x212a-0x2130  |
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 693c42392886..c1edfa55b071 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -37,6 +37,7 @@
 #include "qla_bsg.h"
 #include "qla_nx.h"
 #include "qla_nx2.h"
+#include "qla_nvme.h"
 #define QLA2XXX_DRIVER_NAME"qla2xxx"
 #define QLA2XXX_APIDEV "ql2xapidev"
 #define QLA2XXX_MANUFACTURER   "QLogic Corporation"
@@ -423,6 +424,7 @@ struct srb_iocb {
int rsp_len;
dma_addr_t cmd_dma;
dma_addr_t rsp_dma;
+   enum nvmefc_fcp_datadir dir;
uint32_t dl;
uint32_t timeout_sec;
} nvme;
@@ -452,6 +454,7 @@ struct srb_iocb {
 #define SRB_NACK_PRLI  17
 #define SRB_NACK_LOGO  18
 #define SRB_NVME_CMD   19
+#define SRB_NVME_LS20
 #define SRB_PRLI_CMD   21
 
 enum {
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 6fbee11c1a18..fc2c03bda731 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -10,6 +10,32 @@
 #include 
 
 /*
+ * Global functions prototype in qla_nvme.c source file.
+ */
+extern void qla_nvme_register_hba(scsi_qla_host_t *);
+extern int  qla_nvme_register_remote(scsi_qla_host_t *, fc_port_t *);
+extern void qla_nvme_delete(scsi_qla_host_t *);
+extern void qla_nvme_abort(struct qla_hw_data *, srb_t *sp);
+extern void qla_nvme_unregister_remote_port(struct work_struct *);
+extern int qla_nvme_wait_on_rport_del(fc_port_t *);
+extern void qla_nvme_abort_all(fc_port_t *);
+extern int qla_nvme_post_cmd(struct nvme_fc_local_port *,
+struct nvme_fc_remote_port *, void *, struct nvmefc_fcp_req *);
+extern int qla_nvme_alloc_queue(struct nvme_fc_local_port *, unsigned int,
+u16, void **);
+extern int qla_nvme_hba_scan(scsi_qla_host_t *);
+extern void qla_nvme_ls_abort(struct nvme_fc_local_port *,
+struct nvme_fc_remote_port *, struct nvmefc_ls_req *);
+extern int qla_nvme_ls_req(struct nvme_fc_local_port *,
+struct nvme_fc_remote_port *, struct nvmefc_ls_req *);
+extern void qla_nvme_poll(struct nvme_fc_local_port *, void *);
+extern int qla2x00_start_nvme_mq(srb_t *);
+extern void