On Fri, Sep 19, 2014 at 9:00 PM, Bart Van Assche wrote:
> Improve performance by using multiple RDMA/RC channels per SCSI host
> for communicating with an SRP target.
>
> Signed-off-by: Bart Van Assche
> ---
> Documentation/ABI/stable/sysfs-driver-ib_srp | 25 +-
> drivers/infiniband/ulp/srp/ib_srp.c | 337
> ---
> drivers/infiniband/ulp/srp/ib_srp.h | 20 +-
> 3 files changed, 287 insertions(+), 95 deletions(-)
>
> diff --git a/Documentation/ABI/stable/sysfs-driver-ib_srp
> b/Documentation/ABI/stable/sysfs-driver-ib_srp
> index b9688de..d5a459e 100644
> --- a/Documentation/ABI/stable/sysfs-driver-ib_srp
> +++ b/Documentation/ABI/stable/sysfs-driver-ib_srp
> @@ -55,12 +55,12 @@ Description:Interface for making ib_srp connect
> to a new target.
> only safe with partial memory descriptor list support
> enabled
> (allow_ext_sg=1).
> * comp_vector, a number in the range 0..n-1 specifying the
> - MSI-X completion vector. Some HCA's allocate multiple (n)
> - MSI-X vectors per HCA port. If the IRQ affinity masks of
> - these interrupts have been configured such that each MSI-X
> - interrupt is handled by a different CPU then the comp_vector
> - parameter can be used to spread the SRP completion workload
> - over multiple CPU's.
> + MSI-X completion vector of the first RDMA channel. Some
> + HCA's allocate multiple (n) MSI-X vectors per HCA port. If
> + the IRQ affinity masks of these interrupts have been
> + configured such that each MSI-X interrupt is handled by a
> + different CPU then the comp_vector parameter can be used to
> + spread the SRP completion workload over multiple CPU's.
> * tl_retry_count, a number in the range 2..7 specifying the
> IB RC retry count.
> * queue_size, the maximum number of commands that the
> @@ -88,6 +88,13 @@ Description: Whether ib_srp is allowed to include a
> partial memory
> descriptor list in an SRP_CMD when communicating with an SRP
> target.
>
> +What: /sys/class/scsi_host/host/ch_count
> +Date: November 1, 2014
> +KernelVersion: 3.18
> +Contact: linux-rdma@vger.kernel.org
> +Description: Number of RDMA channels used for communication with the SRP
> + target.
> +
> What: /sys/class/scsi_host/host/cmd_sg_entries
> Date: May 19, 2011
> KernelVersion: 2.6.39
> @@ -95,6 +102,12 @@ Contact:linux-rdma@vger.kernel.org
> Description: Maximum number of data buffer descriptors that may be sent to
> the target in a single SRP_CMD request.
>
> +What: /sys/class/scsi_host/host/comp_vector
> +Date: September 2, 2013
> +KernelVersion: 3.11
> +Contact: linux-rdma@vger.kernel.org
> +Description: Completion vector used for the first RDMA channel.
> +
> What: /sys/class/scsi_host/host/dgid
> Date: June 17, 2006
> KernelVersion: 2.6.17
> diff --git a/drivers/infiniband/ulp/srp/ib_srp.c
> b/drivers/infiniband/ulp/srp/ib_srp.c
> index 9feeea1..58ca618 100644
> --- a/drivers/infiniband/ulp/srp/ib_srp.c
> +++ b/drivers/infiniband/ulp/srp/ib_srp.c
> @@ -123,6 +123,16 @@ MODULE_PARM_DESC(dev_loss_tmo,
> " if fast_io_fail_tmo has not been set. \"off\" means that"
> " this functionality is disabled.");
>
> +static unsigned ch_count;
> +module_param(ch_count, uint, 0444);
> +MODULE_PARM_DESC(ch_count,
> +"Number of RDMA channels to use for communication with an
> SRP"
> +" target. Using more than one channel improves performance"
> +" if the HCA supports multiple completion vectors. The"
> +" default value is the minimum of four times the number of"
> +" online CPU sockets and the number of completion vectors"
> +" supported by the HCA.");
> +
> static void srp_add_one(struct ib_device *device);
> static void srp_remove_one(struct ib_device *device);
> static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
> @@ -556,17 +566,32 @@ err:
> * Note: this function may be called without srp_alloc_iu_bufs() having been
> * invoked. Hence the ch->[rt]x_ring checks.
> */
> -static void srp_free_ch_ib(struct srp_rdma_ch *ch)
> +static void srp_free_ch_ib(struct srp_target_port *target,
> + struct srp_rdma_ch *ch)
> {
> - struct srp_target_port *target = ch->target;
> struct srp_device *dev = target->srp_host->srp_dev;
> int i;
>
> + if (!ch->target)
> + return;
> +
> + /*
> +* Avoid that the SCSI error handler tries to use this channel after
> +* it has been freed. Th
