On Tue, Dec 5, 2023 at 11:47 AM Dragos Tatulea <dtatu...@nvidia.com> wrote:
>
> Instead of tearing down and setting up vq resources, use vq
> suspend/resume during .set_map to speed things up a bit.
>
> The vq mr is updated with the new mapping while the vqs are suspended.
>
> If the device doesn't support resumable vqs, do the old teardown and
> setup dance.
>
> Signed-off-by: Dragos Tatulea <dtatu...@nvidia.com>
> Reviewed-by: Gal Pressman <g...@nvidia.com>
> Acked-by: Eugenio Pérez <epere...@redhat.com>
I didn't ack it, but I'm ok with it, so:
Acked-by: Eugenio Pérez <epere...@redhat.com>
Thanks!
> ---
> drivers/vdpa/mlx5/net/mlx5_vnet.c | 46 ++++++++++++++++++++++++------
> include/linux/mlx5/mlx5_ifc_vdpa.h | 1 +
> 2 files changed, 39 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c
> b/drivers/vdpa/mlx5/net/mlx5_vnet.c
> index d6c8506cec8f..6a21223d97a8 100644
> --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
> +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
> @@ -1206,6 +1206,7 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev,
> {
> int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in);
> u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {};
> + struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
> bool state_change = false;
> void *obj_context;
> void *cmd_hdr;
> @@ -1255,6 +1256,24 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev,
> if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX)
> MLX5_SET(virtio_net_q_object, obj_context, hw_used_index,
> mvq->used_idx);
>
> + if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY) {
> + struct mlx5_vdpa_mr *mr =
> mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
> +
> + if (mr)
> + MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, mr->mkey);
> + else
> + mvq->modified_fields &=
> ~MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY;
> + }
> +
> + if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY) {
> + struct mlx5_vdpa_mr *mr =
> mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
> +
> + if (mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev,
> desc_group_mkey_supported))
> + MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, mr->mkey);
> + else
> + mvq->modified_fields &=
> ~MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
> + }
> +
> MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select,
> mvq->modified_fields);
> err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
> if (err)
> @@ -2784,24 +2803,35 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev
> *mvdev,
> unsigned int asid)
> {
> struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
> + bool teardown = !is_resumable(ndev);
> int err;
>
> suspend_vqs(ndev);
> - err = save_channels_info(ndev);
> - if (err)
> - return err;
> + if (teardown) {
> + err = save_channels_info(ndev);
> + if (err)
> + return err;
>
> - teardown_driver(ndev);
> + teardown_driver(ndev);
> + }
>
> mlx5_vdpa_update_mr(mvdev, new_mr, asid);
>
> + for (int i = 0; i < ndev->cur_num_vqs; i++)
> + ndev->vqs[i].modified_fields |=
> MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY |
> +
> MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
> +
> if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) || mvdev->suspended)
> return 0;
>
> - restore_channels_info(ndev);
> - err = setup_driver(mvdev);
> - if (err)
> - return err;
> + if (teardown) {
> + restore_channels_info(ndev);
> + err = setup_driver(mvdev);
> + if (err)
> + return err;
> + }
> +
> + resume_vqs(ndev);
>
> return 0;
> }
> diff --git a/include/linux/mlx5/mlx5_ifc_vdpa.h
> b/include/linux/mlx5/mlx5_ifc_vdpa.h
> index 32e712106e68..40371c916cf9 100644
> --- a/include/linux/mlx5/mlx5_ifc_vdpa.h
> +++ b/include/linux/mlx5/mlx5_ifc_vdpa.h
> @@ -148,6 +148,7 @@ enum {
> MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS = (u64)1 << 6,
> MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_AVAIL_IDX = (u64)1 << 7,
> MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX = (u64)1 << 8,
> + MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY = (u64)1 << 11,
> MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY = (u64)1 << 14,
> };
>
> --
> 2.42.0
>
