From: Xie Yongji <xieyon...@bytedance.com>
To support reconnecting after restart or crash, VDUSE backend
might need to resubmit inflight I/Os. This stores the metadata
such as the index of inflight I/O's descriptors to a shm file so
that VDUSE backend can restore them during reconnecting.
Signed-off-by: Xie Yongji <xieyon...@bytedance.com>
Message-Id: <20220523084611.91-9-xieyon...@bytedance.com>
Reviewed-by: Stefan Hajnoczi <stefa...@redhat.com>
Signed-off-by: Kevin Wolf <kw...@redhat.com>
---
subprojects/libvduse/libvduse.h | 12 ++
block/export/vduse-blk.c | 19 ++-
subprojects/libvduse/libvduse.c | 235 +++++++++++++++++++++++++++++++-
3 files changed, 260 insertions(+), 6 deletions(-)
diff --git a/subprojects/libvduse/libvduse.h b/subprojects/libvduse/libvduse.h
index 6c2fe98213..32f19e7b48 100644
--- a/subprojects/libvduse/libvduse.h
+++ b/subprojects/libvduse/libvduse.h
@@ -173,6 +173,18 @@ int vduse_dev_update_config(VduseDev *dev, uint32_t size,
*/
int vduse_dev_setup_queue(VduseDev *dev, int index, int max_size);
+/**
+ * vduse_set_reconnect_log_file:
+ * @dev: VDUSE device
+ * @file: filename of reconnect log
+ *
+ * Specify the file to store log for reconnecting. It should
+ * be called before vduse_dev_setup_queue().
+ *
+ * Returns: 0 on success, -errno on failure.
+ */
+int vduse_set_reconnect_log_file(VduseDev *dev, const char *filename);
+
/**
* vduse_dev_create_by_fd:
* @fd: passed file descriptor
diff --git a/block/export/vduse-blk.c b/block/export/vduse-blk.c
index cab1904234..251d73c841 100644
--- a/block/export/vduse-blk.c
+++ b/block/export/vduse-blk.c
@@ -30,6 +30,7 @@ typedef struct VduseBlkExport {
VirtioBlkHandler handler;
VduseDev *dev;
uint16_t num_queues;
+ char *recon_file;
unsigned int inflight;
} VduseBlkExport;
@@ -125,6 +126,8 @@ static void vduse_blk_enable_queue(VduseDev *dev,
VduseVirtq *vq)
aio_set_fd_handler(vblk_exp->export.ctx, vduse_queue_get_fd(vq),
true, on_vduse_vq_kick, NULL, NULL, NULL, vq);
+ /* Make sure we don't miss any kick afer reconnecting */
+ eventfd_write(vduse_queue_get_fd(vq), 1);
}
static void vduse_blk_disable_queue(VduseDev *dev, VduseVirtq *vq)
@@ -306,6 +309,15 @@ static int vduse_blk_exp_create(BlockExport *exp,
BlockExportOptions *opts,
return -ENOMEM;
}
+ vblk_exp->recon_file = g_strdup_printf("%s/vduse-blk-%s",
+ g_get_tmp_dir(), exp->id);
+ if (vduse_set_reconnect_log_file(vblk_exp->dev, vblk_exp->recon_file)) {
+ error_setg(errp, "failed to set reconnect log file");
+ vduse_dev_destroy(vblk_exp->dev);
+ g_free(vblk_exp->recon_file);
+ return -EINVAL;
+ }
+
for (i = 0; i < num_queues; i++) {
vduse_dev_setup_queue(vblk_exp->dev, i, queue_size);
}
@@ -324,11 +336,16 @@ static int vduse_blk_exp_create(BlockExport *exp,
BlockExportOptions *opts,
static void vduse_blk_exp_delete(BlockExport *exp)
{
VduseBlkExport *vblk_exp = container_of(exp, VduseBlkExport, export);
+ int ret;
blk_remove_aio_context_notifier(exp->blk, blk_aio_attached, blk_aio_detach,
vblk_exp);
blk_set_dev_ops(exp->blk, NULL, NULL);
- vduse_dev_destroy(vblk_exp->dev);
+ ret = vduse_dev_destroy(vblk_exp->dev);
+ if (ret != -EBUSY) {
+ unlink(vblk_exp->recon_file);
+ }
+ g_free(vblk_exp->recon_file);
}
static void vduse_blk_exp_request_shutdown(BlockExport *exp)
diff --git a/subprojects/libvduse/libvduse.c b/subprojects/libvduse/libvduse.c
index 78e1e5cf90..9a2bcec282 100644
--- a/subprojects/libvduse/libvduse.c
+++ b/subprojects/libvduse/libvduse.c
@@ -42,6 +42,8 @@
#define VDUSE_VQ_ALIGN 4096
#define MAX_IOVA_REGIONS 256
+#define LOG_ALIGNMENT 64
+
/* Round number down to multiple */
#define ALIGN_DOWN(n, m) ((n) / (m) * (m))
@@ -52,6 +54,31 @@
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
+typedef struct VduseDescStateSplit {
+ uint8_t inflight;
+ uint8_t padding[5];
+ uint16_t next;
+ uint64_t counter;
+} VduseDescStateSplit;
+
+typedef struct VduseVirtqLogInflight {
+ uint64_t features;
+ uint16_t version;
+ uint16_t desc_num;
+ uint16_t last_batch_head;
+ uint16_t used_idx;
+ VduseDescStateSplit desc[];
+} VduseVirtqLogInflight;
+
+typedef struct VduseVirtqLog {
+ VduseVirtqLogInflight inflight;
+} VduseVirtqLog;
+
+typedef struct VduseVirtqInflightDesc {
+ uint16_t index;
+ uint64_t counter;
+} VduseVirtqInflightDesc;
+
typedef struct VduseRing {
unsigned int num;
uint64_t desc_addr;
@@ -74,6 +101,10 @@ struct VduseVirtq {
bool ready;
int fd;
VduseDev *dev;
+ VduseVirtqInflightDesc *resubmit_list;
+ uint16_t resubmit_num;
+ uint64_t counter;
+ VduseVirtqLog *log;
};
typedef struct VduseIovaRegion {
@@ -97,8 +128,36 @@ struct VduseDev {
int fd;
int ctrl_fd;
void *priv;
+ void *log;
};
+static inline size_t vduse_vq_log_size(uint16_t queue_size)
+{
+ return ALIGN_UP(sizeof(VduseDescStateSplit) * queue_size +
+ sizeof(VduseVirtqLogInflight), LOG_ALIGNMENT);
+}
+
+static void *vduse_log_get(const char *filename, size_t size)
+{
+ void *ptr = MAP_FAILED;
+ int fd;
+
+ fd = open(filename, O_RDWR | O_CREAT, 0600);
+ if (fd == -1) {
+ return MAP_FAILED;
+ }
+
+ if (ftruncate(fd, size) == -1) {
+ goto out;
+ }
+
+ ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+
+out:
+ close(fd);
+ return ptr;
+}
+
static inline bool has_feature(uint64_t features, unsigned int fbit)
{
assert(fbit < 64);
@@ -149,6 +208,105 @@ static int vduse_inject_irq(VduseDev *dev, int index)
return ioctl(dev->fd, VDUSE_VQ_INJECT_IRQ, &index);
}
+static int inflight_desc_compare(const void *a, const void *b)
+{
+ VduseVirtqInflightDesc *desc0 = (VduseVirtqInflightDesc *)a,
+ *desc1 = (VduseVirtqInflightDesc *)b;
+
+ if (desc1->counter > desc0->counter &&
+ (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2) {
+ return 1;
+ }
+
+ return -1;
+}
+
+static int vduse_queue_check_inflights(VduseVirtq *vq)
+{
+ int i = 0;
+ VduseDev *dev = vq->dev;
+
+ vq->used_idx = le16toh(vq->vring.used->idx);
+ vq->resubmit_num = 0;
+ vq->resubmit_list = NULL;
+ vq->counter = 0;
+
+ if (unlikely(vq->log->inflight.used_idx != vq->used_idx)) {
+ if (vq->log->inflight.last_batch_head > VIRTQUEUE_MAX_SIZE) {
+ return -1;
+ }
+
+ vq->log->inflight.desc[vq->log->inflight.last_batch_head].inflight = 0;
+
+ barrier();
+
+ vq->log->inflight.used_idx = vq->used_idx;
+ }
+
+ for (i = 0; i < vq->log->inflight.desc_num; i++) {
+ if (vq->log->inflight.desc[i].inflight == 1) {
+ vq->inuse++;
+ }
+ }
+
+ vq->shadow_avail_idx = vq->last_avail_idx = vq->inuse + vq->used_idx;
+
+ if (vq->inuse) {
+ vq->resubmit_list = calloc(vq->inuse, sizeof(VduseVirtqInflightDesc));
+ if (!vq->resubmit_list) {
+ return -1;
+ }
+
+ for (i = 0; i < vq->log->inflight.desc_num; i++) {
+ if (vq->log->inflight.desc[i].inflight) {
+ vq->resubmit_list[vq->resubmit_num].index = i;
+ vq->resubmit_list[vq->resubmit_num].counter =
+ vq->log->inflight.desc[i].counter;
+ vq->resubmit_num++;
+ }
+ }
+
+ if (vq->resubmit_num > 1) {
+ qsort(vq->resubmit_list, vq->resubmit_num,
+ sizeof(VduseVirtqInflightDesc), inflight_desc_compare);
+ }
+ vq->counter = vq->resubmit_list[0].counter + 1;
+ }
+
+ vduse_inject_irq(dev, vq->index);
+
+ return 0;
+}
+
+static int vduse_queue_inflight_get(VduseVirtq *vq, int desc_idx)
+{
+ vq->log->inflight.desc[desc_idx].counter = vq->counter++;
+
+ barrier();
+
+ vq->log->inflight.desc[desc_idx].inflight = 1;
+
+ return 0;
+}
+
+static int vduse_queue_inflight_pre_put(VduseVirtq *vq, int desc_idx)
+{
+ vq->log->inflight.last_batch_head = desc_idx;
+
+ return 0;
+}
+
+static int vduse_queue_inflight_post_put(VduseVirtq *vq, int desc_idx)
+{
+ vq->log->inflight.desc[desc_idx].inflight = 0;
+
+ barrier();
+
+ vq->log->inflight.used_idx = vq->used_idx;
+
+ return 0;
+}
+
static void vduse_iova_remove_region(VduseDev *dev, uint64_t start,
uint64_t last)
{
@@ -579,11 +737,24 @@ void *vduse_queue_pop(VduseVirtq *vq, size_t sz)
unsigned int head;
VduseVirtqElement *elem;
VduseDev *dev = vq->dev;
+ int i;
if (unlikely(!vq->vring.avail)) {
return NULL;
}
+ if (unlikely(vq->resubmit_list && vq->resubmit_num > 0)) {
+ i = (--vq->resubmit_num);
+ elem = vduse_queue_map_desc(vq, vq->resubmit_list[i].index, sz);
+
+ if (!vq->resubmit_num) {
+ free(vq->resubmit_list);
+ vq->resubmit_list = NULL;
+ }
+
+ return elem;
+ }
+
if (vduse_queue_empty(vq)) {
return NULL;
}
@@ -611,6 +782,8 @@ void *vduse_queue_pop(VduseVirtq *vq, size_t sz)
vq->inuse++;
+ vduse_queue_inflight_get(vq, head);
+
return elem;
}
@@ -668,7 +841,9 @@ void vduse_queue_push(VduseVirtq *vq, const
VduseVirtqElement *elem,
unsigned int len)
{
vduse_queue_fill(vq, elem, len, 0);
+ vduse_queue_inflight_pre_put(vq, elem->index);
vduse_queue_flush(vq, 1);
+ vduse_queue_inflight_post_put(vq, elem->index);
}
static int vduse_queue_update_vring(VduseVirtq *vq, uint64_t desc_addr,
@@ -747,12 +922,15 @@ static void vduse_queue_enable(VduseVirtq *vq)
}
vq->fd = fd;
- vq->shadow_avail_idx = vq->last_avail_idx = vq_info.split.avail_index;
- vq->inuse = 0;
- vq->used_idx = 0;
vq->signalled_used_valid = false;
vq->ready = true;
+ if (vduse_queue_check_inflights(vq)) {
+ fprintf(stderr, "Failed to check inflights for vq[%d]\n", vq->index);
+ close(fd);
+ return;
+ }
+
dev->ops->enable_queue(dev, vq);
}
@@ -802,11 +980,15 @@ static void vduse_dev_start_dataplane(VduseDev *dev)
static void vduse_dev_stop_dataplane(VduseDev *dev)
{
+ size_t log_size = dev->num_queues * vduse_vq_log_size(VIRTQUEUE_MAX_SIZE);
int i;
for (i = 0; i < dev->num_queues; i++) {
vduse_queue_disable(&dev->vqs[i]);
}
+ if (dev->log) {
+ memset(dev->log, 0, log_size);
+ }
dev->features = 0;
vduse_iova_remove_region(dev, 0, ULONG_MAX);
}
@@ -915,6 +1097,30 @@ int vduse_dev_setup_queue(VduseDev *dev, int index, int
max_size)
return -errno;
}
+ vduse_queue_enable(vq);
+
+ return 0;
+}
+
+int vduse_set_reconnect_log_file(VduseDev *dev, const char *filename)
+{
+
+ size_t log_size = dev->num_queues * vduse_vq_log_size(VIRTQUEUE_MAX_SIZE);
+ void *log;
+ int i;
+
+ dev->log = log = vduse_log_get(filename, log_size);
+ if (log == MAP_FAILED) {
+ fprintf(stderr, "Failed to get vduse log\n");
+ return -EINVAL;
+ }
+
+ for (i = 0; i < dev->num_queues; i++) {
+ dev->vqs[i].log = log;
+ dev->vqs[i].log->inflight.desc_num = VIRTQUEUE_MAX_SIZE;
+ log = (void *)((char *)log + vduse_vq_log_size(VIRTQUEUE_MAX_SIZE));
+ }
+
return 0;
}
@@ -959,6 +1165,12 @@ static int vduse_dev_init(VduseDev *dev, const char *name,
return -errno;
}
+ if (ioctl(fd, VDUSE_DEV_GET_FEATURES, &dev->features)) {
+ fprintf(stderr, "Failed to get features: %s\n", strerror(errno));
+ close(fd);
+ return -errno;
+ }
+
dev_name = strdup(name);
if (!dev_name) {
close(fd);
@@ -1003,6 +1215,12 @@ VduseDev *vduse_dev_create_by_fd(int fd, uint16_t
num_queues,
return NULL;
}
+ if (ioctl(fd, VDUSE_DEV_GET_FEATURES, &dev->features)) {
+ fprintf(stderr, "Failed to get features: %s\n", strerror(errno));
+ free(dev);
+ return NULL;
+ }
+
ret = vduse_dev_init_vqs(dev, num_queues);
if (ret) {
fprintf(stderr, "Failed to init vqs\n");
@@ -1102,7 +1320,7 @@ VduseDev *vduse_dev_create(const char *name, uint32_t
device_id,
ret = ioctl(ctrl_fd, VDUSE_CREATE_DEV, dev_config);
free(dev_config);
- if (ret < 0) {
+ if (ret && errno != EEXIST) {
fprintf(stderr, "Failed to create vduse device %s: %s\n",
name, strerror(errno));
goto err_dev;
@@ -1129,8 +1347,15 @@ err_ctrl:
int vduse_dev_destroy(VduseDev *dev)
{
- int ret = 0;
+ size_t log_size = dev->num_queues * vduse_vq_log_size(VIRTQUEUE_MAX_SIZE);
+ int i, ret = 0;
+ if (dev->log) {
+ munmap(dev->log, log_size);
+ }
+ for (i = 0; i < dev->num_queues; i++) {
+ free(dev->vqs[i].resubmit_list);
+ }
free(dev->vqs);
if (dev->fd >= 0) {
close(dev->fd);
--
2.35.3
