Make BlockExportType.iothread an alternate between a single-thread variant 'str' and a multi-threading variant '[str]'.
In contrast to the single-thread setting, the multi-threading setting will not change the BDS's context (and so is incompatible with the fixed-iothread setting), but instead just pass a list to the export driver, with which it can do whatever it wants. Currently no export driver supports multi-threading, so they all return an error when receiving such a list. Suggested-by: Kevin Wolf <[email protected]> Acked-by: Markus Armbruster <[email protected]> Reviewed-by: Stefan Hajnoczi <[email protected]> Signed-off-by: Hanna Czenczek <[email protected]> --- qapi/block-export.json | 36 ++++++++++++++++++--- include/block/export.h | 12 +++++-- block/export/export.c | 48 +++++++++++++++++++++++++--- block/export/fuse.c | 7 ++++ block/export/vduse-blk.c | 7 ++++ block/export/vhost-user-blk-server.c | 8 +++++ nbd/server.c | 6 ++++ 7 files changed, 113 insertions(+), 11 deletions(-) diff --git a/qapi/block-export.json b/qapi/block-export.json index 076954ef1a..160cd2e3ca 100644 --- a/qapi/block-export.json +++ b/qapi/block-export.json @@ -363,14 +363,16 @@ # to the export before completion is signalled. (since: 5.2; # default: false) # -# @iothread: The name of the iothread object where the export will -# run. The default is to use the thread currently associated with -# the block node. (since: 5.2) +# @iothread: The name(s) of one or more iothread object(s) where the +# export will run. The default is to use the thread currently +# associated with the block node. (since: 5.2; multi-threading +# since 10.1) # # @fixed-iothread: True prevents the block node from being moved to # another thread while the export is active. If true and # @iothread is given, export creation fails if the block node -# cannot be moved to the iothread. The default is false. +# cannot be moved to the iothread. Must not be true when giving +# multiple iothreads for @iothread. The default is false. # (since: 5.2) # # @allow-inactive: If true, the export allows the exported node to be @@ -387,7 +389,7 @@ 'base': { 'type': 'BlockExportType', 'id': 'str', '*fixed-iothread': 'bool', - '*iothread': 'str', + '*iothread': 'BlockExportIothreads', 'node-name': 'str', '*writable': 'bool', '*writethrough': 'bool', @@ -403,6 +405,30 @@ 'if': 'CONFIG_VDUSE_BLK_EXPORT' } } } +## +# @BlockExportIothreads: +# +# Specify a single or multiple I/O threads in which to run a block +# export's I/O. +# +# @single: Run the export's I/O in the given single I/O thread. +# +# @multi: Use multi-threading across the given set of I/O threads, +# which must not be empty. Note: Passing a single I/O thread via +# this variant is still treated as multi-threading, which is +# different from using the @single variant. In particular, even +# if there only is a single I/O thread in the set, export types +# that do not support multi-threading will generally reject this +# variant, and BlockExportOptions.fixed-iothread is always +# incompatible with it. +# +# Since: 10.1 +## +{ 'alternate': 'BlockExportIothreads', + 'data': { + 'single': 'str', + 'multi': ['str'] } } + ## # @block-export-add: # diff --git a/include/block/export.h b/include/block/export.h index 4bd9531d4d..ca45da928c 100644 --- a/include/block/export.h +++ b/include/block/export.h @@ -32,8 +32,16 @@ typedef struct BlockExportDriver { /* True if the export type supports running on an inactive node */ bool supports_inactive; - /* Creates and starts a new block export */ - int (*create)(BlockExport *, BlockExportOptions *, Error **); + /* + * Creates and starts a new block export. + * + * If the user passed a set of I/O threads for multi-threading, @multithread + * is a list of the @multithread_count corresponding contexts (freed by the + * caller). Note that @exp->ctx has no relation to that list. + */ + int (*create)(BlockExport *exp, BlockExportOptions *opts, + AioContext *const *multithread, size_t multithread_count, + Error **errp); /* * Frees a removed block export. This function is only called after all diff --git a/block/export/export.c b/block/export/export.c index f3bbf11070..b733f269f3 100644 --- a/block/export/export.c +++ b/block/export/export.c @@ -76,16 +76,26 @@ BlockExport *blk_exp_add(BlockExportOptions *export, Error **errp) { bool fixed_iothread = export->has_fixed_iothread && export->fixed_iothread; bool allow_inactive = export->has_allow_inactive && export->allow_inactive; + bool multithread = export->iothread && + export->iothread->type == QTYPE_QLIST; const BlockExportDriver *drv; BlockExport *exp = NULL; BlockDriverState *bs; BlockBackend *blk = NULL; AioContext *ctx; + AioContext **multithread_ctxs = NULL; + size_t multithread_count = 0; uint64_t perm; int ret; GLOBAL_STATE_CODE(); + if (fixed_iothread && multithread) { + error_setg(errp, + "Cannot use fixed-iothread for a multi-threaded export"); + return NULL; + } + if (!id_wellformed(export->id)) { error_setg(errp, "Invalid block export id"); return NULL; @@ -116,14 +126,16 @@ BlockExport *blk_exp_add(BlockExportOptions *export, Error **errp) ctx = bdrv_get_aio_context(bs); - if (export->iothread) { + /* Move the BDS to the target I/O thread, if it is a single one */ + if (export->iothread && !multithread) { + const char *iothread_id = export->iothread->u.single; IOThread *iothread; AioContext *new_ctx; Error **set_context_errp; - iothread = iothread_by_id(export->iothread); + iothread = iothread_by_id(iothread_id); if (!iothread) { - error_setg(errp, "iothread \"%s\" not found", export->iothread); + error_setg(errp, "iothread \"%s\" not found", iothread_id); goto fail; } @@ -137,6 +149,32 @@ BlockExport *blk_exp_add(BlockExportOptions *export, Error **errp) } else if (fixed_iothread) { goto fail; } + } else if (multithread) { + strList *iothread_list = export->iothread->u.multi; + size_t i; + + multithread_count = 0; + for (strList *e = iothread_list; e; e = e->next) { + multithread_count++; + } + + if (multithread_count == 0) { + error_setg(errp, "The set of I/O threads must not be empty"); + return NULL; + } + + multithread_ctxs = g_new(AioContext *, multithread_count); + i = 0; + for (strList *e = iothread_list; e; e = e->next) { + IOThread *iothread = iothread_by_id(e->value); + + if (!iothread) { + error_setg(errp, "iothread \"%s\" not found", e->value); + goto fail; + } + multithread_ctxs[i++] = iothread_get_aio_context(iothread); + } + assert(i == multithread_count); } bdrv_graph_rdlock_main_loop(); @@ -195,7 +233,7 @@ BlockExport *blk_exp_add(BlockExportOptions *export, Error **errp) .blk = blk, }; - ret = drv->create(exp, export, errp); + ret = drv->create(exp, export, multithread_ctxs, multithread_count, errp); if (ret < 0) { goto fail; } @@ -203,6 +241,7 @@ BlockExport *blk_exp_add(BlockExportOptions *export, Error **errp) assert(exp->blk != NULL); QLIST_INSERT_HEAD(&block_exports, exp, next); + g_free(multithread_ctxs); return exp; fail: @@ -214,6 +253,7 @@ fail: g_free(exp->id); g_free(exp); } + g_free(multithread_ctxs); return NULL; } diff --git a/block/export/fuse.c b/block/export/fuse.c index b60affe86b..aa4f1c0307 100644 --- a/block/export/fuse.c +++ b/block/export/fuse.c @@ -260,6 +260,8 @@ static const BlockDevOps fuse_export_blk_dev_ops = { static int fuse_export_create(BlockExport *blk_exp, BlockExportOptions *blk_exp_args, + AioContext *const *multithread, + size_t mt_count, Error **errp) { ERRP_GUARD(); /* ensure clean-up even with error_fatal */ @@ -269,6 +271,11 @@ static int fuse_export_create(BlockExport *blk_exp, assert(blk_exp_args->type == BLOCK_EXPORT_TYPE_FUSE); + if (multithread) { + error_setg(errp, "FUSE export does not support multi-threading"); + return -EINVAL; + } + /* For growable and writable exports, take the RESIZE permission */ if (args->growable || blk_exp_args->writable) { uint64_t blk_perm, blk_shared_perm; diff --git a/block/export/vduse-blk.c b/block/export/vduse-blk.c index 8af13b7f0b..10dc673c56 100644 --- a/block/export/vduse-blk.c +++ b/block/export/vduse-blk.c @@ -267,6 +267,7 @@ static const BlockDevOps vduse_block_ops = { }; static int vduse_blk_exp_create(BlockExport *exp, BlockExportOptions *opts, + AioContext *const *multithread, size_t mt_count, Error **errp) { VduseBlkExport *vblk_exp = container_of(exp, VduseBlkExport, export); @@ -302,6 +303,12 @@ static int vduse_blk_exp_create(BlockExport *exp, BlockExportOptions *opts, return -EINVAL; } } + + if (multithread) { + error_setg(errp, "vduse-blk export does not support multi-threading"); + return -EINVAL; + } + vblk_exp->num_queues = num_queues; vblk_exp->handler.blk = exp->blk; vblk_exp->handler.serial = g_strdup(vblk_opts->serial ?: ""); diff --git a/block/export/vhost-user-blk-server.c b/block/export/vhost-user-blk-server.c index a4d54e824f..e89422bb85 100644 --- a/block/export/vhost-user-blk-server.c +++ b/block/export/vhost-user-blk-server.c @@ -316,6 +316,7 @@ static const BlockDevOps vu_blk_dev_ops = { }; static int vu_blk_exp_create(BlockExport *exp, BlockExportOptions *opts, + AioContext *const *multithread, size_t mt_count, Error **errp) { VuBlkExport *vexp = container_of(exp, VuBlkExport, export); @@ -341,6 +342,13 @@ static int vu_blk_exp_create(BlockExport *exp, BlockExportOptions *opts, error_setg(errp, "num-queues must be greater than 0"); return -EINVAL; } + + if (multithread) { + error_setg(errp, + "vhost-user-blk export does not support multi-threading"); + return -EINVAL; + } + vexp->handler.blk = exp->blk; vexp->handler.serial = g_strdup("vhost_user_blk"); vexp->handler.logical_block_size = logical_block_size; diff --git a/nbd/server.c b/nbd/server.c index acec0487a8..620097c58c 100644 --- a/nbd/server.c +++ b/nbd/server.c @@ -1795,6 +1795,7 @@ static const BlockDevOps nbd_block_ops = { }; static int nbd_export_create(BlockExport *blk_exp, BlockExportOptions *exp_args, + AioContext *const *multithread, size_t mt_count, Error **errp) { NBDExport *exp = container_of(blk_exp, NBDExport, common); @@ -1831,6 +1832,11 @@ static int nbd_export_create(BlockExport *blk_exp, BlockExportOptions *exp_args, return -EEXIST; } + if (multithread) { + error_setg(errp, "NBD export does not support multi-threading"); + return -EINVAL; + } + size = blk_getlength(blk); if (size < 0) { error_setg_errno(errp, -size, -- 2.53.0
