> -----Original Message-----
> From: Bryan Zhang <bryan.zh...@bytedance.com>
> Sent: Wednesday, March 27, 2024 6:42 AM
> To: qemu-devel@nongnu.org
> Cc: pet...@redhat.com; faro...@suse.de; Liu, Yuan1 <yuan1....@intel.com>;
> berra...@redhat.com; Zou, Nanhai <nanhai....@intel.com>;
> hao.xi...@linux.dev; Bryan Zhang <bryan.zh...@bytedance.com>
> Subject: [PATCH v2 2/5] migration: Add migration parameters for QATzip
>
> Adds support for migration parameters to control QATzip compression
> level and to enable/disable software fallback when QAT hardware is
> unavailable. This is a preparatory commit for a subsequent commit that
> will actually use QATzip compression.
>
> Signed-off-by: Bryan Zhang <bryan.zh...@bytedance.com>
> Signed-off-by: Hao Xiang <hao.xi...@linux.dev>
> ---
> Revision: This commit now includes a parameter for controlling software
> fallback. Fallback is generally intended to be disabled, but having this
> option available enables using software fallback for testing.
>
> This commit also now has some glue code to properly set parameters.
>
> migration/migration-hmp-cmds.c | 8 +++++
> migration/options.c | 57 ++++++++++++++++++++++++++++++++++
> migration/options.h | 2 ++
> qapi/migration.json | 35 +++++++++++++++++++++
> 4 files changed, 102 insertions(+)
>
> diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-
> cmds.c
> index 99b49df5dd..4bd23ba14d 100644
> --- a/migration/migration-hmp-cmds.c
> +++ b/migration/migration-hmp-cmds.c
> @@ -630,6 +630,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const
> QDict *qdict)
> p->has_multifd_zlib_level = true;
> visit_type_uint8(v, param, &p->multifd_zlib_level, &err);
> break;
> + case MIGRATION_PARAMETER_MULTIFD_QATZIP_LEVEL:
> + p->has_multifd_qatzip_level = true;
> + visit_type_uint8(v, param, &p->multifd_qatzip_level, &err);
> + break;
> + case MIGRATION_PARAMETER_MULTIFD_QATZIP_SW_FALLBACK:
> + p->has_multifd_qatzip_sw_fallback = true;
> + visit_type_bool(v, param, &p->multifd_qatzip_sw_fallback, &err);
> + break;
> case MIGRATION_PARAMETER_MULTIFD_ZSTD_LEVEL:
> p->has_multifd_zstd_level = true;
> visit_type_uint8(v, param, &p->multifd_zstd_level, &err);
> diff --git a/migration/options.c b/migration/options.c
> index 3e3e0b93b4..1316ea605a 100644
> --- a/migration/options.c
> +++ b/migration/options.c
> @@ -62,6 +62,15 @@
> #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
> /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
> #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
> +/*
> + * 1: best speed, ... 9: best compress ratio
> + * There is some nuance here. Refer to QATzip documentation to understand
> + * the mapping of QATzip levels to standard deflate levels.
> + */
> +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL 1
> +/* QATzip's SW fallback implementation is extremely slow, so avoid
> fallback */
> +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK false
> +
> /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
> #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
Hi Bryan
The default compression level may be set higher, such as 6. I checked QAT
throughput
If the data size is less than or equal to 64K, level 1 has much better
throughput
performance than level 6 and level 9. But if the data size is greater than
128K, little
change in throughput, and the default MULTIFD_PACKET_SIZE is 512K, you can have
a try
to use a high compression level, to get better compression performance without
affecting
throughput.
In addition, if you change MULTIFD_PACKET_SIZE to 64K, you may have better
throughput
with more multifd threads
> @@ -143,6 +152,12 @@ Property migration_properties[] = {
> DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
> parameters.multifd_zlib_level,
> DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
> + DEFINE_PROP_UINT8("multifd-qatzip-level", MigrationState,
> + parameters.multifd_qatzip_level,
> + DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL),
> + DEFINE_PROP_BOOL("multifd-qatzip-sw-fallback", MigrationState,
> + parameters.multifd_qatzip_sw_fallback,
> + DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK),
> DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
> parameters.multifd_zstd_level,
> DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
> @@ -861,6 +876,20 @@ int migrate_multifd_zlib_level(void)
> return s->parameters.multifd_zlib_level;
> }
>
> +int migrate_multifd_qatzip_level(void)
> +{
> + MigrationState *s = migrate_get_current();
> +
> + return s->parameters.multifd_qatzip_level;
> +}
> +
> +bool migrate_multifd_qatzip_sw_fallback(void)
> +{
> + MigrationState *s = migrate_get_current();
> +
> + return s->parameters.multifd_qatzip_sw_fallback;
> +}
> +
> int migrate_multifd_zstd_level(void)
> {
> MigrationState *s = migrate_get_current();
> @@ -983,6 +1012,11 @@ MigrationParameters
> *qmp_query_migrate_parameters(Error **errp)
> params->multifd_compression = s->parameters.multifd_compression;
> params->has_multifd_zlib_level = true;
> params->multifd_zlib_level = s->parameters.multifd_zlib_level;
> + params->has_multifd_qatzip_level = true;
> + params->multifd_qatzip_level = s->parameters.multifd_qatzip_level;
> + params->has_multifd_qatzip_sw_fallback = true;
> + params->multifd_qatzip_sw_fallback =
> + s->parameters.multifd_qatzip_sw_fallback;
> params->has_multifd_zstd_level = true;
> params->multifd_zstd_level = s->parameters.multifd_zstd_level;
> params->has_xbzrle_cache_size = true;
> @@ -1038,6 +1072,8 @@ void migrate_params_init(MigrationParameters
> *params)
> params->has_multifd_channels = true;
> params->has_multifd_compression = true;
> params->has_multifd_zlib_level = true;
> + params->has_multifd_qatzip_level = true;
> + params->has_multifd_qatzip_sw_fallback = true;
> params->has_multifd_zstd_level = true;
> params->has_xbzrle_cache_size = true;
> params->has_max_postcopy_bandwidth = true;
> @@ -1147,6 +1183,14 @@ bool migrate_params_check(MigrationParameters
> *params, Error **errp)
> return false;
> }
>
> + if (params->has_multifd_qatzip_level &&
> + ((params->multifd_qatzip_level > 9) ||
> + (params->multifd_qatzip_level < 1))) {
> + error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
> "multifd_qatzip_level",
> + "a value between 1 and 9");
> + return false;
> + }
> +
> if (params->has_multifd_zstd_level &&
> (params->multifd_zstd_level > 20)) {
> error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
> "multifd_zstd_level",
> @@ -1312,6 +1356,12 @@ static void
> migrate_params_test_apply(MigrateSetParameters *params,
> if (params->has_multifd_compression) {
> dest->multifd_compression = params->multifd_compression;
> }
> + if (params->has_multifd_qatzip_level) {
> + dest->multifd_qatzip_level = params->multifd_qatzip_level;
> + }
> + if (params->has_multifd_qatzip_sw_fallback) {
> + dest->multifd_qatzip_sw_fallback = params-
> >multifd_qatzip_sw_fallback;
> + }
> if (params->has_xbzrle_cache_size) {
> dest->xbzrle_cache_size = params->xbzrle_cache_size;
> }
> @@ -1447,6 +1497,13 @@ static void
> migrate_params_apply(MigrateSetParameters *params, Error **errp)
> if (params->has_multifd_compression) {
> s->parameters.multifd_compression = params->multifd_compression;
> }
> + if (params->has_multifd_qatzip_level) {
> + s->parameters.multifd_qatzip_level = params-
> >multifd_qatzip_level;
> + }
> + if (params->has_multifd_qatzip_sw_fallback) {
> + s->parameters.multifd_qatzip_sw_fallback =
> + params->multifd_qatzip_sw_fallback;
> + }
> if (params->has_xbzrle_cache_size) {
> s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
> xbzrle_cache_resize(params->xbzrle_cache_size, errp);
> diff --git a/migration/options.h b/migration/options.h
> index 246c160aee..94aee24d97 100644
> --- a/migration/options.h
> +++ b/migration/options.h
> @@ -87,6 +87,8 @@ MigMode migrate_mode(void);
> int migrate_multifd_channels(void);
> MultiFDCompression migrate_multifd_compression(void);
> int migrate_multifd_zlib_level(void);
> +int migrate_multifd_qatzip_level(void);
> +bool migrate_multifd_qatzip_sw_fallback(void);
> int migrate_multifd_zstd_level(void);
> uint8_t migrate_throttle_trigger_threshold(void);
> const char *migrate_tls_authz(void);
> diff --git a/qapi/migration.json b/qapi/migration.json
> index 0b33a71ab4..66ea6d32fc 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -853,6 +853,16 @@
> # speed, and 9 means best compression ratio which will consume
> # more CPU. Defaults to 1. (Since 5.0)
> #
> +# @multifd-qatzip-level: Set the compression level to be used in live
> +# migration. The level is an integer between 1 and 9, where 1 means
> +# the best compression speed, and 9 means the best compression
> +# ratio which will consume more CPU. Defaults to 1.
> +#
> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware
> +# is unavailable. Defaults to false. Software fallback performance
> +# is very poor compared to regular zlib, so be cautious about
> +# enabling this option.
> +#
> # @multifd-zstd-level: Set the compression level to be used in live
> # migration, the compression level is an integer between 0 and 20,
> # where 0 means no compression, 1 means the best compression
> @@ -915,6 +925,7 @@
> 'xbzrle-cache-size', 'max-postcopy-bandwidth',
> 'max-cpu-throttle', 'multifd-compression',
> 'multifd-zlib-level', 'multifd-zstd-level',
> + 'multifd-qatzip-level', 'multifd-qatzip-sw-fallback',
> 'block-bitmap-mapping',
> { 'name': 'x-vcpu-dirty-limit-period', 'features':
> ['unstable'] },
> 'vcpu-dirty-limit',
> @@ -1045,6 +1056,16 @@
> # speed, and 9 means best compression ratio which will consume
> # more CPU. Defaults to 1. (Since 5.0)
> #
> +# @multifd-qatzip-level: Set the compression level to be used in live
> +# migration. The level is an integer between 1 and 9, where 1 means
> +# the best compression speed, and 9 means the best compression
> +# ratio which will consume more CPU. Defaults to 1.
> +#
> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware
> +# is unavailable. Defaults to false. Software fallback performance
> +# is very poor compared to regular zlib, so be cautious about
> +# enabling this option.
> +#
> # @multifd-zstd-level: Set the compression level to be used in live
> # migration, the compression level is an integer between 0 and 20,
> # where 0 means no compression, 1 means the best compression
> @@ -1125,6 +1146,8 @@
> '*max-cpu-throttle': 'uint8',
> '*multifd-compression': 'MultiFDCompression',
> '*multifd-zlib-level': 'uint8',
> + '*multifd-qatzip-level': 'uint8',
> + '*multifd-qatzip-sw-fallback': 'bool',
> '*multifd-zstd-level': 'uint8',
> '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ],
> '*x-vcpu-dirty-limit-period': { 'type': 'uint64',
> @@ -1273,6 +1296,16 @@
> # speed, and 9 means best compression ratio which will consume
> # more CPU. Defaults to 1. (Since 5.0)
> #
> +# @multifd-qatzip-level: Set the compression level to be used in live
> +# migration. The level is an integer between 1 and 9, where 1 means
> +# the best compression speed, and 9 means the best compression
> +# ratio which will consume more CPU. Defaults to 1.
> +#
> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware
> +# is unavailable. Defaults to false. Software fallback performance
> +# is very poor compared to regular zlib, so be cautious about
> +# enabling this option.
> +#
> # @multifd-zstd-level: Set the compression level to be used in live
> # migration, the compression level is an integer between 0 and 20,
> # where 0 means no compression, 1 means the best compression
> @@ -1350,6 +1383,8 @@
> '*max-cpu-throttle': 'uint8',
> '*multifd-compression': 'MultiFDCompression',
> '*multifd-zlib-level': 'uint8',
> + '*multifd-qatzip-level': 'uint8',
> + '*multifd-qatzip-sw-fallback': 'bool',
> '*multifd-zstd-level': 'uint8',
> '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ],
> '*x-vcpu-dirty-limit-period': { 'type': 'uint64',
> --
> 2.30.2
