On Tue, Oct 27, 2026 at 02:56:04AM +0000, Kevin Loughlin wrote:
> Allow individual virtqueues to override the device-wide DMA mapping
> operations via virtqueue_set_map_ops(). When custom per-virtqueue
> mapping operations are configured, vring_map_one_sg(), vring_unmap_one(),
> vring_mapping_error(), and the DMA sync helpers dispatch through
> vq->map_ops instead of the parent virtio_device DMA API path.
>
> Replace vring_map_single() with vring_map_indirect() and
> vring_unmap_indirect() using the device-level mapping path (vdev->map
> or dma_map_page_attrs()) so that indirect descriptor tables continue to
> be DMA-mapped for host VMM access even when a virtqueue overrides buffer
> mapping via vq->map_ops.
>
> Signed-off-by: Kevin Loughlin <[email protected]>
Please, don't. The automatic mappings inside virtio core are there
to keep simple things, simple.
Everyone doing something funky with mappings should use the premapped API.
> ---
> drivers/virtio/virtio_ring.c | 136 ++++++++++++++++++++++++-----------
> include/linux/virtio.h | 4 ++
> 2 files changed, 98 insertions(+), 42 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index db678f5a80e0..9c6a1e9f0a0b 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -261,6 +261,7 @@ struct vring_virtqueue {
> bool we_own_ring;
>
> union virtio_map map;
> + const struct virtio_map_ops *map_ops;
>
> #ifdef DEBUG
> /* They're supposed to lock for us. */
> @@ -477,16 +478,32 @@ static struct device *vring_dma_dev(const struct
> vring_virtqueue *vq)
> return vq->map.dma_dev;
> }
>
> +static inline const struct virtio_map_ops *vring_get_map_ops(const struct
> vring_virtqueue *vq)
> +{
> + if (vq->map_ops)
> + return vq->map_ops;
> + return vq->vq.vdev->map;
> +}
> +
> +void virtqueue_set_map_ops(struct virtqueue *_vq,
> + const struct virtio_map_ops *map_ops)
> +{
> + struct vring_virtqueue *vq = to_vvq(_vq);
> +
> + vq->map_ops = map_ops;
> +}
> +EXPORT_SYMBOL_GPL(virtqueue_set_map_ops);
> +
> static int vring_mapping_error(const struct vring_virtqueue *vq,
> dma_addr_t addr)
> {
> - struct virtio_device *vdev = vq->vq.vdev;
> + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
>
> if (!vq->use_map_api)
> return 0;
>
> - if (vdev->map)
> - return vdev->map->mapping_error(vq->map, addr);
> + if (map_ops)
> + return map_ops->mapping_error(vq->map, addr);
> else
> return dma_mapping_error(vring_dma_dev(vq), addr);
> }
> @@ -530,15 +547,45 @@ static int vring_map_one_sg(const struct
> vring_virtqueue *vq, struct scatterlist
> return 0;
> }
>
> -static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
> - void *cpu_addr, size_t size,
> - enum dma_data_direction direction)
> +static int vring_map_indirect(const struct vring_virtqueue *vq, void
> *cpu_addr,
> + size_t size, dma_addr_t *addr)
> {
> - if (!vq->use_map_api)
> - return (dma_addr_t)virt_to_phys(cpu_addr);
> + struct virtio_device *vdev = vq->vq.vdev;
> +
> + if (!vq->use_map_api) {
> + kmsan_handle_dma(virt_to_phys(cpu_addr), size, DMA_TO_DEVICE);
> + *addr = (dma_addr_t)virt_to_phys(cpu_addr);
> + return 0;
> + }
> +
> + if (vdev->map) {
> + *addr = vdev->map->map_page(vq->map, virt_to_page(cpu_addr),
> + offset_in_page(cpu_addr), size,
> + DMA_TO_DEVICE, 0);
> + if (vdev->map->mapping_error(vq->map, *addr))
> + return -ENOMEM;
> + return 0;
> + }
> +
> + *addr = dma_map_page_attrs(vring_dma_dev(vq), virt_to_page(cpu_addr),
> + offset_in_page(cpu_addr), size,
> + DMA_TO_DEVICE, 0);
> + if (dma_mapping_error(vring_dma_dev(vq), *addr))
> + return -ENOMEM;
>
> - return virtqueue_map_single_attrs(&vq->vq, cpu_addr,
> - size, direction, 0);
> + return 0;
> +}
> +
> +static void vring_unmap_indirect(const struct vring_virtqueue *vq,
> + dma_addr_t addr, size_t size)
> +{
> + struct virtio_device *vdev = vq->vq.vdev;
> +
> + if (vdev->map)
> + vdev->map->unmap_page(vq->map, addr, size, DMA_TO_DEVICE, 0);
> + else
> + dma_unmap_page_attrs(vring_dma_dev(vq), addr, size,
> + DMA_TO_DEVICE, 0);
> }
>
> static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
> @@ -576,6 +623,8 @@ static unsigned int vring_unmap_one_split(const struct
> vring_virtqueue *vq,
> if (flags & VRING_DESC_F_INDIRECT) {
> if (!vq->use_map_api)
> goto out;
> + vring_unmap_indirect(vq, extra->addr, extra->len);
> + goto out;
> } else if (!vring_need_unmap_buffer(vq, extra))
> goto out;
>
> @@ -761,10 +810,11 @@ static inline int virtqueue_add_split(struct
> vring_virtqueue *vq,
>
> if (indirect) {
> /* Now that the indirect table is filled in, map it. */
> - dma_addr_t addr = vring_map_single(
> - vq, desc, total_sg * sizeof(struct vring_desc),
> - DMA_TO_DEVICE);
> - if (vring_mapping_error(vq, addr))
> + dma_addr_t addr;
> +
> + if (vring_map_indirect(vq, desc,
> + total_sg * sizeof(struct vring_desc),
> + &addr))
> goto unmap_release;
>
> virtqueue_add_desc_split(vq, vq->split.vring.desc,
> @@ -1370,6 +1420,7 @@ static struct virtqueue
> *__vring_new_virtqueue_split(unsigned int index,
> #endif
> vq->map = map;
> vq->use_map_api = vring_use_map_api(vdev);
> + vq->map_ops = NULL;
>
> vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
> !context;
> @@ -1488,6 +1539,8 @@ static void vring_unmap_extra_packed(const struct
> vring_virtqueue *vq,
> if (flags & VRING_DESC_F_INDIRECT) {
> if (!vq->use_map_api)
> return;
> + vring_unmap_indirect(vq, extra->addr, extra->len);
> + return;
> } else if (!vring_need_unmap_buffer(vq, extra))
> return;
>
> @@ -1585,10 +1638,9 @@ static int virtqueue_add_indirect_packed(struct
> vring_virtqueue *vq,
> }
>
> /* Now that the indirect table is filled in, map it. */
> - addr = vring_map_single(vq, desc,
> - total_sg * sizeof(struct vring_packed_desc),
> - DMA_TO_DEVICE);
> - if (vring_mapping_error(vq, addr))
> + if (vring_map_indirect(vq, desc,
> + total_sg * sizeof(struct vring_packed_desc),
> + &addr))
> goto unmap_release;
>
> vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
> @@ -2620,6 +2672,7 @@ static struct virtqueue
> *__vring_new_virtqueue_packed(unsigned int index,
> #endif
> vq->map = map;
> vq->use_map_api = vring_use_map_api(vdev);
> + vq->map_ops = NULL;
>
> vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
> !context;
> @@ -3064,7 +3117,7 @@ struct device *virtqueue_dma_dev(struct virtqueue *_vq)
> {
> struct vring_virtqueue *vq = to_vvq(_vq);
>
> - if (vq->use_map_api && !_vq->vdev->map)
> + if (vq->use_map_api && !vring_get_map_ops(vq))
> return vq->map.dma_dev;
> else
> return NULL;
> @@ -3809,18 +3862,17 @@ EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent);
> */
> dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
> struct page *page,
> - unsigned long offset,
> - size_t size,
> + unsigned long offset, size_t size,
> enum dma_data_direction dir,
> unsigned long attrs)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct virtio_device *vdev = _vq->vdev;
> + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
>
> - if (vdev->map)
> - return vdev->map->map_page(vq->map,
> - page, offset, size,
> - dir, attrs);
> + if (map_ops)
> + return map_ops->map_page(vq->map,
> + page, offset, size,
> + dir, attrs);
>
> return dma_map_page_attrs(vring_dma_dev(vq),
> page, offset, size,
> @@ -3842,11 +3894,11 @@ void virtqueue_unmap_page_attrs(const struct
> virtqueue *_vq,
> unsigned long attrs)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct virtio_device *vdev = _vq->vdev;
> + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
>
> - if (vdev->map)
> - vdev->map->unmap_page(vq->map,
> - map_handle, size, dir, attrs);
> + if (map_ops)
> + map_ops->unmap_page(vq->map,
> + map_handle, size, dir, attrs);
> else
> dma_unmap_page_attrs(vring_dma_dev(vq), map_handle,
> size, dir, attrs);
> @@ -3941,13 +3993,13 @@ EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error);
> bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct virtio_device *vdev = _vq->vdev;
> + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
>
> if (!vq->use_map_api)
> return false;
>
> - if (vdev->map)
> - return vdev->map->need_sync(vq->map, addr);
> + if (map_ops)
> + return map_ops->need_sync(vq->map, addr);
> else
> return dma_need_sync(vring_dma_dev(vq), addr);
> }
> @@ -3971,14 +4023,14 @@ void virtqueue_map_sync_single_range_for_cpu(const
> struct virtqueue *_vq,
> enum dma_data_direction dir)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct virtio_device *vdev = _vq->vdev;
> + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
>
> if (!vq->use_map_api)
> return;
>
> - if (vdev->map)
> - vdev->map->sync_single_for_cpu(vq->map,
> - addr + offset, size, dir);
> + if (map_ops)
> + map_ops->sync_single_for_cpu(vq->map,
> + addr + offset, size, dir);
> else
> dma_sync_single_range_for_cpu(vring_dma_dev(vq),
> addr, offset, size, dir);
> @@ -4002,15 +4054,15 @@ void virtqueue_map_sync_single_range_for_device(const
> struct virtqueue *_vq,
> enum dma_data_direction dir)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct virtio_device *vdev = _vq->vdev;
> + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
>
> if (!vq->use_map_api)
> return;
>
> - if (vdev->map)
> - vdev->map->sync_single_for_device(vq->map,
> - addr + offset,
> - size, dir);
> + if (map_ops)
> + map_ops->sync_single_for_device(vq->map,
> + addr + offset,
> + size, dir);
> else
> dma_sync_single_range_for_device(vring_dma_dev(vq), addr,
> offset, size, dir);
> diff --git a/include/linux/virtio.h b/include/linux/virtio.h
> index f923e42cfd01..20fe333d6778 100644
> --- a/include/linux/virtio.h
> +++ b/include/linux/virtio.h
> @@ -44,6 +44,7 @@ struct virtqueue {
> };
>
> struct vduse_vq_group;
> +struct virtio_map_ops;
>
> union virtio_map {
> /* Device that performs DMA */
> @@ -294,6 +295,9 @@ void virtqueue_map_free_coherent(struct virtio_device
> *vdev,
> size_t size, void *vaddr,
> dma_addr_t dma_handle);
>
> +void virtqueue_set_map_ops(struct virtqueue *_vq,
> + const struct virtio_map_ops *map_ops);
> +
> dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
> struct page *page,
> unsigned long offset,
> --
> 2.56.0.307.g5fe7bd680c-goog
