Allow individual virtqueues to override the device-wide DMA mapping operations via virtqueue_set_map_ops(). When custom per-virtqueue mapping operations are configured, vring_map_one_sg(), vring_unmap_one(), vring_mapping_error(), and the DMA sync helpers dispatch through vq->map_ops instead of the parent virtio_device DMA API path.
Replace vring_map_single() with vring_map_indirect() and vring_unmap_indirect() using the device-level mapping path (vdev->map or dma_map_page_attrs()) so that indirect descriptor tables continue to be DMA-mapped for host VMM access even when a virtqueue overrides buffer mapping via vq->map_ops. Signed-off-by: Kevin Loughlin <[email protected]> --- drivers/virtio/virtio_ring.c | 136 ++++++++++++++++++++++++----------- include/linux/virtio.h | 4 ++ 2 files changed, 98 insertions(+), 42 deletions(-) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index db678f5a80e0..9c6a1e9f0a0b 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -261,6 +261,7 @@ struct vring_virtqueue { bool we_own_ring; union virtio_map map; + const struct virtio_map_ops *map_ops; #ifdef DEBUG /* They're supposed to lock for us. */ @@ -477,16 +478,32 @@ static struct device *vring_dma_dev(const struct vring_virtqueue *vq) return vq->map.dma_dev; } +static inline const struct virtio_map_ops *vring_get_map_ops(const struct vring_virtqueue *vq) +{ + if (vq->map_ops) + return vq->map_ops; + return vq->vq.vdev->map; +} + +void virtqueue_set_map_ops(struct virtqueue *_vq, + const struct virtio_map_ops *map_ops) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + vq->map_ops = map_ops; +} +EXPORT_SYMBOL_GPL(virtqueue_set_map_ops); + static int vring_mapping_error(const struct vring_virtqueue *vq, dma_addr_t addr) { - struct virtio_device *vdev = vq->vq.vdev; + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq); if (!vq->use_map_api) return 0; - if (vdev->map) - return vdev->map->mapping_error(vq->map, addr); + if (map_ops) + return map_ops->mapping_error(vq->map, addr); else return dma_mapping_error(vring_dma_dev(vq), addr); } @@ -530,15 +547,45 @@ static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist return 0; } -static dma_addr_t vring_map_single(const struct vring_virtqueue *vq, - void *cpu_addr, size_t size, - enum dma_data_direction direction) +static int vring_map_indirect(const struct vring_virtqueue *vq, void *cpu_addr, + size_t size, dma_addr_t *addr) { - if (!vq->use_map_api) - return (dma_addr_t)virt_to_phys(cpu_addr); + struct virtio_device *vdev = vq->vq.vdev; + + if (!vq->use_map_api) { + kmsan_handle_dma(virt_to_phys(cpu_addr), size, DMA_TO_DEVICE); + *addr = (dma_addr_t)virt_to_phys(cpu_addr); + return 0; + } + + if (vdev->map) { + *addr = vdev->map->map_page(vq->map, virt_to_page(cpu_addr), + offset_in_page(cpu_addr), size, + DMA_TO_DEVICE, 0); + if (vdev->map->mapping_error(vq->map, *addr)) + return -ENOMEM; + return 0; + } + + *addr = dma_map_page_attrs(vring_dma_dev(vq), virt_to_page(cpu_addr), + offset_in_page(cpu_addr), size, + DMA_TO_DEVICE, 0); + if (dma_mapping_error(vring_dma_dev(vq), *addr)) + return -ENOMEM; - return virtqueue_map_single_attrs(&vq->vq, cpu_addr, - size, direction, 0); + return 0; +} + +static void vring_unmap_indirect(const struct vring_virtqueue *vq, + dma_addr_t addr, size_t size) +{ + struct virtio_device *vdev = vq->vq.vdev; + + if (vdev->map) + vdev->map->unmap_page(vq->map, addr, size, DMA_TO_DEVICE, 0); + else + dma_unmap_page_attrs(vring_dma_dev(vq), addr, size, + DMA_TO_DEVICE, 0); } static void virtqueue_init(struct vring_virtqueue *vq, u32 num) @@ -576,6 +623,8 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, if (flags & VRING_DESC_F_INDIRECT) { if (!vq->use_map_api) goto out; + vring_unmap_indirect(vq, extra->addr, extra->len); + goto out; } else if (!vring_need_unmap_buffer(vq, extra)) goto out; @@ -761,10 +810,11 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq, if (indirect) { /* Now that the indirect table is filled in, map it. */ - dma_addr_t addr = vring_map_single( - vq, desc, total_sg * sizeof(struct vring_desc), - DMA_TO_DEVICE); - if (vring_mapping_error(vq, addr)) + dma_addr_t addr; + + if (vring_map_indirect(vq, desc, + total_sg * sizeof(struct vring_desc), + &addr)) goto unmap_release; virtqueue_add_desc_split(vq, vq->split.vring.desc, @@ -1370,6 +1420,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index, #endif vq->map = map; vq->use_map_api = vring_use_map_api(vdev); + vq->map_ops = NULL; vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && !context; @@ -1488,6 +1539,8 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, if (flags & VRING_DESC_F_INDIRECT) { if (!vq->use_map_api) return; + vring_unmap_indirect(vq, extra->addr, extra->len); + return; } else if (!vring_need_unmap_buffer(vq, extra)) return; @@ -1585,10 +1638,9 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq, } /* Now that the indirect table is filled in, map it. */ - addr = vring_map_single(vq, desc, - total_sg * sizeof(struct vring_packed_desc), - DMA_TO_DEVICE); - if (vring_mapping_error(vq, addr)) + if (vring_map_indirect(vq, desc, + total_sg * sizeof(struct vring_packed_desc), + &addr)) goto unmap_release; vq->packed.vring.desc[head].addr = cpu_to_le64(addr); @@ -2620,6 +2672,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index, #endif vq->map = map; vq->use_map_api = vring_use_map_api(vdev); + vq->map_ops = NULL; vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && !context; @@ -3064,7 +3117,7 @@ struct device *virtqueue_dma_dev(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - if (vq->use_map_api && !_vq->vdev->map) + if (vq->use_map_api && !vring_get_map_ops(vq)) return vq->map.dma_dev; else return NULL; @@ -3809,18 +3862,17 @@ EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent); */ dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, struct page *page, - unsigned long offset, - size_t size, + unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { const struct vring_virtqueue *vq = to_vvq(_vq); - struct virtio_device *vdev = _vq->vdev; + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq); - if (vdev->map) - return vdev->map->map_page(vq->map, - page, offset, size, - dir, attrs); + if (map_ops) + return map_ops->map_page(vq->map, + page, offset, size, + dir, attrs); return dma_map_page_attrs(vring_dma_dev(vq), page, offset, size, @@ -3842,11 +3894,11 @@ void virtqueue_unmap_page_attrs(const struct virtqueue *_vq, unsigned long attrs) { const struct vring_virtqueue *vq = to_vvq(_vq); - struct virtio_device *vdev = _vq->vdev; + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq); - if (vdev->map) - vdev->map->unmap_page(vq->map, - map_handle, size, dir, attrs); + if (map_ops) + map_ops->unmap_page(vq->map, + map_handle, size, dir, attrs); else dma_unmap_page_attrs(vring_dma_dev(vq), map_handle, size, dir, attrs); @@ -3941,13 +3993,13 @@ EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error); bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr) { const struct vring_virtqueue *vq = to_vvq(_vq); - struct virtio_device *vdev = _vq->vdev; + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq); if (!vq->use_map_api) return false; - if (vdev->map) - return vdev->map->need_sync(vq->map, addr); + if (map_ops) + return map_ops->need_sync(vq->map, addr); else return dma_need_sync(vring_dma_dev(vq), addr); } @@ -3971,14 +4023,14 @@ void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq, enum dma_data_direction dir) { const struct vring_virtqueue *vq = to_vvq(_vq); - struct virtio_device *vdev = _vq->vdev; + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq); if (!vq->use_map_api) return; - if (vdev->map) - vdev->map->sync_single_for_cpu(vq->map, - addr + offset, size, dir); + if (map_ops) + map_ops->sync_single_for_cpu(vq->map, + addr + offset, size, dir); else dma_sync_single_range_for_cpu(vring_dma_dev(vq), addr, offset, size, dir); @@ -4002,15 +4054,15 @@ void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq, enum dma_data_direction dir) { const struct vring_virtqueue *vq = to_vvq(_vq); - struct virtio_device *vdev = _vq->vdev; + const struct virtio_map_ops *map_ops = vring_get_map_ops(vq); if (!vq->use_map_api) return; - if (vdev->map) - vdev->map->sync_single_for_device(vq->map, - addr + offset, - size, dir); + if (map_ops) + map_ops->sync_single_for_device(vq->map, + addr + offset, + size, dir); else dma_sync_single_range_for_device(vring_dma_dev(vq), addr, offset, size, dir); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index f923e42cfd01..20fe333d6778 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -44,6 +44,7 @@ struct virtqueue { }; struct vduse_vq_group; +struct virtio_map_ops; union virtio_map { /* Device that performs DMA */ @@ -294,6 +295,9 @@ void virtqueue_map_free_coherent(struct virtio_device *vdev, size_t size, void *vaddr, dma_addr_t dma_handle); +void virtqueue_set_map_ops(struct virtqueue *_vq, + const struct virtio_map_ops *map_ops); + dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, struct page *page, unsigned long offset, -- 2.56.0.307.g5fe7bd680c-goog
