Handle VIRTIO_BLK_T_WRITE_ZEROES for parity with virtio-blk. The range is zeroed with __blkdev_issue_zeroout(), which uses WRITE ZEROES of the backend or falls back to writing zero pages. The UNMAP flag maps to the absence of BLKDEV_ZERO_NOUNMAP. The range is issued in chunks of at most 16 WRITE ZEROES bios and 1 GiB, or 4 MiB ones when zero pages are written, since each chunk is built in the vhost worker.
The VMM must advertise blk_size equal to the backend logical block size, max_write_zeroes_sectors aligned to it and at most 8388607, and max_write_zeroes_seg = 1. The guest gets VIRTIO_BLK_S_UNSUPP when VIRTIO_BLK_F_WRITE_ZEROES was not negotiated, for a payload other than exactly one segment or an unknown flag, and VIRTIO_BLK_S_IOERR for a read-only backend and for a range that is unaligned, beyond the capacity or above 8388607 sectors. https://virtuozzo.atlassian.net/browse/VSTOR-141684 Feature: vhost-blk: in-kernel accelerator for virtio-blk guests Signed-off-by: Alexander Ivanov <[email protected]> --- drivers/vhost/blk.c | 178 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 2 deletions(-) diff --git a/drivers/vhost/blk.c b/drivers/vhost/blk.c index eea8cca59cac..e27320651469 100644 --- a/drivers/vhost/blk.c +++ b/drivers/vhost/blk.c @@ -29,7 +29,8 @@ enum { (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_BLK_F_MQ) | - (1ULL << VIRTIO_BLK_F_FLUSH), + (1ULL << VIRTIO_BLK_F_FLUSH) | + (1ULL << VIRTIO_BLK_F_WRITE_ZEROES), }; /* @@ -75,6 +76,10 @@ struct vhost_blk_req { struct iovec status[VHOST_MAX_METADATA_IOV]; sector_t sector; + sector_t nr_sects; + sector_t chunk_sects; + struct block_device *bdev; + unsigned int zeroout_flags; int bi_opf; u16 head; size_t len; @@ -271,6 +276,63 @@ static int vhost_blk_move_bb_to_req(struct vhost_blk_req *req) return 0; } +/* + * The range is zeroed in the vhost worker with vq->mutex held, so it is + * split into chunks of a few bios each, much smaller ones when zero pages + * are written. The chunk sizes are multiples of any logical block size. + */ +#define VHOST_BLK_MAX_WRITE_ZEROES_BIOS 16 +#define VHOST_BLK_MAX_WRITE_ZEROES_SECTORS (SZ_1G >> SECTOR_SHIFT) +#define VHOST_BLK_MAX_ZERO_PAGES_SECTORS (SZ_4M >> SECTOR_SHIFT) + +/* + * A request holds vhost_blk_flush() until its last chunk completes, so the + * whole range is bounded too. No Linux guest sends more, a bio cannot carry + * it. + */ +#define VHOST_BLK_MAX_WRITE_ZEROES_REQ_SECTORS (UINT_MAX >> SECTOR_SHIFT) + +static int vhost_blk_bio_make_zeroes(struct vhost_blk_req *req, + struct block_device *bdev) +{ + sector_t bs_mask = (bdev_logical_block_size(bdev) >> SECTOR_SHIFT) - 1; + sector_t max_sects = (sector_t)bdev_write_zeroes_sectors(bdev) * + VHOST_BLK_MAX_WRITE_ZEROES_BIOS & ~bs_mask; + struct bio *bio = NULL; + int ret = -EOPNOTSUPP; + + req->chunk_sects = min3(req->nr_sects, max_sects, + (sector_t)VHOST_BLK_MAX_WRITE_ZEROES_SECTORS); + /* The limit may appear after it was read, and no bio is built then */ + if (req->chunk_sects) + ret = __blkdev_issue_zeroout(bdev, req->sector, + req->chunk_sects, GFP_KERNEL, &bio, + req->zeroout_flags | + BLKDEV_ZERO_NOFALLBACK); + if (ret == -EOPNOTSUPP) { + req->chunk_sects = min_t(sector_t, req->nr_sects, + VHOST_BLK_MAX_ZERO_PAGES_SECTORS); + ret = __blkdev_issue_zeroout(bdev, req->sector, + req->chunk_sects, GFP_KERNEL, &bio, + req->zeroout_flags); + } + if (ret) + return ret; + + bio->bi_private = req; + bio->bi_end_io = vhost_blk_req_done; + req->bio = req->inline_bio; + req->bio[0] = bio; + + /* Zero pages are plain writes, a failure of them is not retried */ + if (bio_op(bio) != REQ_OP_WRITE_ZEROES) + req->bi_opf = REQ_OP_WRITE; + + atomic_set(&req->bio_nr, 1); + + return 0; +} + static int vhost_blk_bio_make(struct vhost_blk_req *req, struct block_device *bdev) { @@ -284,6 +346,9 @@ static int vhost_blk_bio_make(struct vhost_blk_req *req, if (unlikely(req->bi_opf & REQ_PREFLUSH)) return vhost_blk_bio_make_simple(req, bdev); + if (req->bi_opf == REQ_OP_WRITE_ZEROES) + return vhost_blk_bio_make_zeroes(req, bdev); + /* * Requests with unaligned io vectors are not supported by underlined * block device, let's use an aligned intermediary "bounce buffer" to @@ -451,6 +516,50 @@ static bool vhost_blk_backend_ro(struct file *f) bdev_read_only(I_BDEV(f->f_mapping->host)); } +static int vhost_blk_parse_write_zeroes(struct vhost_blk_req *req, u8 *status) +{ + struct file *f = vhost_vq_get_backend(&req->blk_vq->vq); + struct block_device *bdev = I_BDEV(f->f_mapping->host); + sector_t bs_mask = (bdev_logical_block_size(bdev) >> SECTOR_SHIFT) - 1; + struct virtio_blk_discard_write_zeroes seg; + struct iov_iter iter; + u32 flags; + + if (!vhost_has_feature(&req->blk_vq->vq, VIRTIO_BLK_F_WRITE_ZEROES) || + req->len != sizeof(seg)) { + *status = VIRTIO_BLK_S_UNSUPP; + return 0; + } + + iov_iter_init(&iter, ITER_SOURCE, req->iov, req->iov_nr, req->len); + if (copy_from_iter(&seg, sizeof(seg), &iter) != sizeof(seg)) + return -EFAULT; + + flags = le32_to_cpu(seg.flags); + if (flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) { + *status = VIRTIO_BLK_S_UNSUPP; + return 0; + } + + req->sector = le64_to_cpu(seg.sector); + req->nr_sects = le32_to_cpu(seg.num_sectors); + req->bdev = bdev; + req->zeroout_flags = flags & VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP ? + 0 : BLKDEV_ZERO_NOUNMAP; + + if (vhost_blk_backend_ro(f) || + (req->sector | req->nr_sects) & bs_mask || + req->nr_sects > VHOST_BLK_MAX_WRITE_ZEROES_REQ_SECTORS || + req->sector > bdev_nr_sectors(bdev) || + req->nr_sects > bdev_nr_sectors(bdev) - req->sector) { + *status = VIRTIO_BLK_S_IOERR; + return 0; + } + + *status = VIRTIO_BLK_S_OK; + return 0; +} + static int vhost_blk_req_handle(struct vhost_virtqueue *vq, struct virtio_blk_outhdr *hdr, u16 head, u16 out, u16 in) @@ -475,6 +584,7 @@ static int vhost_blk_req_handle(struct vhost_virtqueue *vq, req->head = head; req->blk = blk; req->sector = vhost64_to_cpu(vq, hdr->sector); + req->nr_sects = 0; req->iov = blk_vq->iov; req->bio_err = 0; in_len = iov_length(vq->iov + out, in); @@ -490,7 +600,7 @@ static int vhost_blk_req_handle(struct vhost_virtqueue *vq, req->status[0].iov_base = st->iov_base + st->iov_len; req->status[0].iov_len = sizeof(status); - if (type == VIRTIO_BLK_T_OUT) { + if (type == VIRTIO_BLK_T_OUT || type == VIRTIO_BLK_T_WRITE_ZEROES) { data_iov = vq->iov; data_nr = out; } else { @@ -538,6 +648,22 @@ static int vhost_blk_req_handle(struct vhost_virtqueue *vq, req->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; ret = vhost_blk_req_submit(req); break; + case VIRTIO_BLK_T_WRITE_ZEROES: + ret = vhost_blk_parse_write_zeroes(req, &status); + if (ret) + break; + if (status == VIRTIO_BLK_S_OK && req->nr_sects) { + req->bi_opf = REQ_OP_WRITE_ZEROES; + ret = vhost_blk_req_submit(req); + if (ret != -EPERM) + break; + status = VIRTIO_BLK_S_IOERR; + } + ret = vhost_blk_set_status(req, status); + if (ret) + break; + vhost_add_used_and_signal(&blk->dev, vq, head, sizeof(status)); + break; case VIRTIO_BLK_T_GET_ID: len = min_t(size_t, strnlen(blk->serial, VIRTIO_BLK_ID_BYTES) + 1, @@ -561,6 +687,51 @@ static int vhost_blk_req_handle(struct vhost_virtqueue *vq, return ret; } +/* + * Some devices clear their WRITE ZEROES limit only after a failed request, + * as blkdev_issue_write_zeroes() expects. Retry such a request with zero + * pages instead of failing it. + */ +static bool vhost_blk_retry_zeroes(struct vhost_blk_req *req) +{ + if (!req->bio_err || req->bi_opf != REQ_OP_WRITE_ZEROES) + return false; + + if (bdev_write_zeroes_sectors(req->bdev)) + return false; + + /* The limit may come back before the bio is built, retry only once */ + req->bi_opf = REQ_OP_WRITE; + if (vhost_blk_bio_make_zeroes(req, req->bdev)) + return false; + + req->bio_err = 0; + vhost_blk_bio_send(req); + return true; +} + +/* Issue the next chunk of a write-zeroes range, if any is left */ +static bool vhost_blk_next_zeroes(struct vhost_blk_req *req) +{ + int ret; + + if (req->bio_err || req->nr_sects <= req->chunk_sects) + return false; + + req->sector += req->chunk_sects; + req->nr_sects -= req->chunk_sects; + + req->bi_opf = REQ_OP_WRITE_ZEROES; + ret = vhost_blk_bio_make_zeroes(req, req->bdev); + if (ret) { + req->bio_err = ret; + return false; + } + + vhost_blk_bio_send(req); + return true; +} + static void vhost_blk_handle_guest_kick(struct vhost_work *work) { struct virtio_blk_outhdr hdr; @@ -661,6 +832,9 @@ static void vhost_blk_handle_host_kick(struct vhost_work *work) if (!blk) blk = req->blk; + if (vhost_blk_retry_zeroes(req) || vhost_blk_next_zeroes(req)) + continue; + if (!req->bio_err && req->bb && req->bi_opf == REQ_OP_READ) { if (vhost_blk_move_bb_to_req(req)) req->bio_err = EINVAL; -- 2.53.0 _______________________________________________ Devel mailing list [email protected] https://lists.openvz.org/mailman/listinfo/devel
