[PATCH 01/13] virtio_ring: support per-virtqueue DMA mapping ops
From: Kevin Loughlin
Date: Mon Oct 26 2026 - 22:59:49 EST
Allow individual virtqueues to override the device-wide DMA mapping
operations via virtqueue_set_map_ops(). When custom per-virtqueue
mapping operations are configured, vring_map_one_sg(), vring_unmap_one(),
vring_mapping_error(), and the DMA sync helpers dispatch through
vq->map_ops instead of the parent virtio_device DMA API path.
Replace vring_map_single() with vring_map_indirect() and
vring_unmap_indirect() using the device-level mapping path (vdev->map
or dma_map_page_attrs()) so that indirect descriptor tables continue to
be DMA-mapped for host VMM access even when a virtqueue overrides buffer
mapping via vq->map_ops.
Signed-off-by: Kevin Loughlin <kevinloughlin@xxxxxxxxxx>
---
drivers/virtio/virtio_ring.c | 136 ++++++++++++++++++++++++-----------
include/linux/virtio.h | 4 ++
2 files changed, 98 insertions(+), 42 deletions(-)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index db678f5a80e0..9c6a1e9f0a0b 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -261,6 +261,7 @@ struct vring_virtqueue {
bool we_own_ring;
union virtio_map map;
+ const struct virtio_map_ops *map_ops;
#ifdef DEBUG
/* They're supposed to lock for us. */
@@ -477,16 +478,32 @@ static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
return vq->map.dma_dev;
}
+static inline const struct virtio_map_ops *vring_get_map_ops(const struct vring_virtqueue *vq)
+{
+ if (vq->map_ops)
+ return vq->map_ops;
+ return vq->vq.vdev->map;
+}
+
+void virtqueue_set_map_ops(struct virtqueue *_vq,
+ const struct virtio_map_ops *map_ops)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+
+ vq->map_ops = map_ops;
+}
+EXPORT_SYMBOL_GPL(virtqueue_set_map_ops);
+
static int vring_mapping_error(const struct vring_virtqueue *vq,
dma_addr_t addr)
{
- struct virtio_device *vdev = vq->vq.vdev;
+ const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
if (!vq->use_map_api)
return 0;
- if (vdev->map)
- return vdev->map->mapping_error(vq->map, addr);
+ if (map_ops)
+ return map_ops->mapping_error(vq->map, addr);
else
return dma_mapping_error(vring_dma_dev(vq), addr);
}
@@ -530,15 +547,45 @@ static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist
return 0;
}
-static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
- void *cpu_addr, size_t size,
- enum dma_data_direction direction)
+static int vring_map_indirect(const struct vring_virtqueue *vq, void *cpu_addr,
+ size_t size, dma_addr_t *addr)
{
- if (!vq->use_map_api)
- return (dma_addr_t)virt_to_phys(cpu_addr);
+ struct virtio_device *vdev = vq->vq.vdev;
+
+ if (!vq->use_map_api) {
+ kmsan_handle_dma(virt_to_phys(cpu_addr), size, DMA_TO_DEVICE);
+ *addr = (dma_addr_t)virt_to_phys(cpu_addr);
+ return 0;
+ }
+
+ if (vdev->map) {
+ *addr = vdev->map->map_page(vq->map, virt_to_page(cpu_addr),
+ offset_in_page(cpu_addr), size,
+ DMA_TO_DEVICE, 0);
+ if (vdev->map->mapping_error(vq->map, *addr))
+ return -ENOMEM;
+ return 0;
+ }
+
+ *addr = dma_map_page_attrs(vring_dma_dev(vq), virt_to_page(cpu_addr),
+ offset_in_page(cpu_addr), size,
+ DMA_TO_DEVICE, 0);
+ if (dma_mapping_error(vring_dma_dev(vq), *addr))
+ return -ENOMEM;
- return virtqueue_map_single_attrs(&vq->vq, cpu_addr,
- size, direction, 0);
+ return 0;
+}
+
+static void vring_unmap_indirect(const struct vring_virtqueue *vq,
+ dma_addr_t addr, size_t size)
+{
+ struct virtio_device *vdev = vq->vq.vdev;
+
+ if (vdev->map)
+ vdev->map->unmap_page(vq->map, addr, size, DMA_TO_DEVICE, 0);
+ else
+ dma_unmap_page_attrs(vring_dma_dev(vq), addr, size,
+ DMA_TO_DEVICE, 0);
}
static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
@@ -576,6 +623,8 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
if (flags & VRING_DESC_F_INDIRECT) {
if (!vq->use_map_api)
goto out;
+ vring_unmap_indirect(vq, extra->addr, extra->len);
+ goto out;
} else if (!vring_need_unmap_buffer(vq, extra))
goto out;
@@ -761,10 +810,11 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
if (indirect) {
/* Now that the indirect table is filled in, map it. */
- dma_addr_t addr = vring_map_single(
- vq, desc, total_sg * sizeof(struct vring_desc),
- DMA_TO_DEVICE);
- if (vring_mapping_error(vq, addr))
+ dma_addr_t addr;
+
+ if (vring_map_indirect(vq, desc,
+ total_sg * sizeof(struct vring_desc),
+ &addr))
goto unmap_release;
virtqueue_add_desc_split(vq, vq->split.vring.desc,
@@ -1370,6 +1420,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
#endif
vq->map = map;
vq->use_map_api = vring_use_map_api(vdev);
+ vq->map_ops = NULL;
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
@@ -1488,6 +1539,8 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
if (flags & VRING_DESC_F_INDIRECT) {
if (!vq->use_map_api)
return;
+ vring_unmap_indirect(vq, extra->addr, extra->len);
+ return;
} else if (!vring_need_unmap_buffer(vq, extra))
return;
@@ -1585,10 +1638,9 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
}
/* Now that the indirect table is filled in, map it. */
- addr = vring_map_single(vq, desc,
- total_sg * sizeof(struct vring_packed_desc),
- DMA_TO_DEVICE);
- if (vring_mapping_error(vq, addr))
+ if (vring_map_indirect(vq, desc,
+ total_sg * sizeof(struct vring_packed_desc),
+ &addr))
goto unmap_release;
vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
@@ -2620,6 +2672,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
#endif
vq->map = map;
vq->use_map_api = vring_use_map_api(vdev);
+ vq->map_ops = NULL;
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
@@ -3064,7 +3117,7 @@ struct device *virtqueue_dma_dev(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
- if (vq->use_map_api && !_vq->vdev->map)
+ if (vq->use_map_api && !vring_get_map_ops(vq))
return vq->map.dma_dev;
else
return NULL;
@@ -3809,18 +3862,17 @@ EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent);
*/
dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
struct page *page,
- unsigned long offset,
- size_t size,
+ unsigned long offset, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
- struct virtio_device *vdev = _vq->vdev;
+ const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
- if (vdev->map)
- return vdev->map->map_page(vq->map,
- page, offset, size,
- dir, attrs);
+ if (map_ops)
+ return map_ops->map_page(vq->map,
+ page, offset, size,
+ dir, attrs);
return dma_map_page_attrs(vring_dma_dev(vq),
page, offset, size,
@@ -3842,11 +3894,11 @@ void virtqueue_unmap_page_attrs(const struct virtqueue *_vq,
unsigned long attrs)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
- struct virtio_device *vdev = _vq->vdev;
+ const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
- if (vdev->map)
- vdev->map->unmap_page(vq->map,
- map_handle, size, dir, attrs);
+ if (map_ops)
+ map_ops->unmap_page(vq->map,
+ map_handle, size, dir, attrs);
else
dma_unmap_page_attrs(vring_dma_dev(vq), map_handle,
size, dir, attrs);
@@ -3941,13 +3993,13 @@ EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error);
bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
- struct virtio_device *vdev = _vq->vdev;
+ const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
if (!vq->use_map_api)
return false;
- if (vdev->map)
- return vdev->map->need_sync(vq->map, addr);
+ if (map_ops)
+ return map_ops->need_sync(vq->map, addr);
else
return dma_need_sync(vring_dma_dev(vq), addr);
}
@@ -3971,14 +4023,14 @@ void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq,
enum dma_data_direction dir)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
- struct virtio_device *vdev = _vq->vdev;
+ const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
if (!vq->use_map_api)
return;
- if (vdev->map)
- vdev->map->sync_single_for_cpu(vq->map,
- addr + offset, size, dir);
+ if (map_ops)
+ map_ops->sync_single_for_cpu(vq->map,
+ addr + offset, size, dir);
else
dma_sync_single_range_for_cpu(vring_dma_dev(vq),
addr, offset, size, dir);
@@ -4002,15 +4054,15 @@ void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq,
enum dma_data_direction dir)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
- struct virtio_device *vdev = _vq->vdev;
+ const struct virtio_map_ops *map_ops = vring_get_map_ops(vq);
if (!vq->use_map_api)
return;
- if (vdev->map)
- vdev->map->sync_single_for_device(vq->map,
- addr + offset,
- size, dir);
+ if (map_ops)
+ map_ops->sync_single_for_device(vq->map,
+ addr + offset,
+ size, dir);
else
dma_sync_single_range_for_device(vring_dma_dev(vq), addr,
offset, size, dir);
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index f923e42cfd01..20fe333d6778 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -44,6 +44,7 @@ struct virtqueue {
};
struct vduse_vq_group;
+struct virtio_map_ops;
union virtio_map {
/* Device that performs DMA */
@@ -294,6 +295,9 @@ void virtqueue_map_free_coherent(struct virtio_device *vdev,
size_t size, void *vaddr,
dma_addr_t dma_handle);
+void virtqueue_set_map_ops(struct virtqueue *_vq,
+ const struct virtio_map_ops *map_ops);
+
dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
struct page *page,
unsigned long offset,
--
2.56.0.307.g5fe7bd680c-goog