diff --git a/drivers/md/dm-vdo/Kconfig b/drivers/md/dm-vdo/Kconfig index 111ecd2c2a24..3aebd1a07076 100644 --- a/drivers/md/dm-vdo/Kconfig +++ b/drivers/md/dm-vdo/Kconfig @@ -5,8 +5,8 @@ config DM_VDO depends on 64BIT depends on BLK_DEV_DM select DM_BUFIO - select LZ4_COMPRESS - select LZ4_DECOMPRESS + select CRYPTO + select CRYPTO_LZ4 help This device mapper target presents a block device with deduplication, compression and thin-provisioning. diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 9476957bfbf4..a390b5573f2f 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only ccflags-y := -I$(src) -I$(src)/indexer +ccflags-y += -DVDO_COMPRESS_STATS_DEBUG obj-$(CONFIG_DM_VDO) += dm-vdo.o diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index ab3ea8337809..06b1d065baca 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -8,16 +8,19 @@ #include #include #include +#include #include #include #include #include #include -#include #include +#include #include +#include #include #include +#include #include "logger.h" #include "memory-alloc.h" @@ -71,6 +74,13 @@ */ static blk_opf_t PASSTHROUGH_FLAGS = (REQ_PRIO | REQ_META | REQ_SYNC | REQ_RAHEAD); +struct vdo_compression_context { + struct crypto_acomp *acomp; + struct acomp_req *req; + struct crypto_wait wait; + struct mutex lock; +}; + /** * DOC: * @@ -1433,6 +1443,106 @@ void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset) vdo_forget(allocation->lock)); } +int make_vdo_compression_context(const char *compressor, + struct vdo_compression_context **context_ptr) +{ + struct vdo_compression_context *context; + int result; + + result = vdo_allocate(1, struct vdo_compression_context, + "compression context", &context); + if (result != VDO_SUCCESS) + return result; + + mutex_init(&context->lock); + context->acomp = crypto_alloc_acomp(compressor, 0, 0); + if (IS_ERR(context->acomp)) { + result = PTR_ERR(context->acomp); + context->acomp = NULL; + vdo_free(context); + return result; + } + + context->req = acomp_request_alloc(context->acomp); + if (context->req == NULL) { + crypto_free_acomp(vdo_forget(context->acomp)); + vdo_free(context); + return -ENOMEM; + } + + crypto_init_wait(&context->wait); + acomp_request_set_callback(context->req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &context->wait); + + *context_ptr = context; + return VDO_SUCCESS; +} + +void free_vdo_compression_context(struct vdo_compression_context *context) +{ + if (context == NULL) + return; + + if (context->req != NULL) + acomp_request_free(vdo_forget(context->req)); + if (context->acomp != NULL) + crypto_free_acomp(vdo_forget(context->acomp)); + vdo_free(context); +} + +static struct vdo_compression_context *get_vdo_compression_context(void) +{ + return vdo_get_work_queue_private_data(); +} + +static int compress_block(struct vdo_compression_context *context, + const char *src, char *dst, u16 *size) +{ + struct scatterlist input, output; + unsigned int dlen = VDO_MAX_COMPRESSED_FRAGMENT_SIZE; + int result; + + sg_init_one(&input, src, VDO_BLOCK_SIZE); + sg_init_one(&output, dst, VDO_MAX_COMPRESSED_FRAGMENT_SIZE); + + mutex_lock(&context->lock); + acomp_request_set_params(context->req, &input, &output, + VDO_BLOCK_SIZE, dlen); + result = crypto_wait_req(crypto_acomp_compress(context->req), + &context->wait); + if (result == 0) { + dlen = context->req->dlen; + if ((dlen > 0) && (dlen < VDO_COMPRESSED_BLOCK_DATA_SIZE)) + *size = dlen; + else + result = -E2BIG; + } + mutex_unlock(&context->lock); + + return result; +} + +static int decompress_block(struct vdo_compression_context *context, + const char *src, u16 src_len, char *dst) +{ + struct scatterlist input, output; + int result; + + sg_init_one(&input, src, src_len); + sg_init_one(&output, dst, VDO_BLOCK_SIZE); + + mutex_lock(&context->lock); + acomp_request_set_params(context->req, &input, &output, + src_len, VDO_BLOCK_SIZE); + result = crypto_wait_req(crypto_acomp_decompress(context->req), + &context->wait); + if ((result == 0) && (context->req->dlen != VDO_BLOCK_SIZE)) + result = -EINVAL; + mutex_unlock(&context->lock); + + return result; +} + /** * uncompress_data_vio() - Uncompress the data a data_vio has just read. * @mapping_state: The mapping state indicating which fragment to decompress. @@ -1441,9 +1551,9 @@ void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset) int uncompress_data_vio(struct data_vio *data_vio, enum block_mapping_state mapping_state, char *buffer) { - int size; u16 fragment_offset, fragment_size; struct compressed_block *block = data_vio->compression.block; + struct vdo_compression_context *context = get_vdo_compression_context(); int result = vdo_get_compressed_block_fragment(mapping_state, block, &fragment_offset, &fragment_size); @@ -1452,10 +1562,15 @@ int uncompress_data_vio(struct data_vio *data_vio, return result; } - size = LZ4_decompress_safe((block->data + fragment_offset), buffer, - fragment_size, VDO_BLOCK_SIZE); - if (size != VDO_BLOCK_SIZE) { - vdo_log_debug("%s: lz4 error", __func__); + if (context == NULL) { + vdo_log_debug("%s: no compression context", __func__); + return VDO_INVALID_FRAGMENT; + } + + result = decompress_block(context, block->data + fragment_offset, + fragment_size, buffer); + if (result != 0) { + vdo_log_debug("%s: decompression error %d", __func__, result); return VDO_INVALID_FRAGMENT; } @@ -1767,19 +1882,31 @@ static void pack_compressed_data(struct vdo_completion *completion) static void compress_data_vio(struct vdo_completion *completion) { struct data_vio *data_vio = as_data_vio(completion); - int size; + struct vdo_compression_context *context = get_vdo_compression_context(); +#ifdef VDO_COMPRESS_STATS_DEBUG + struct vdo *vdo = vdo_from_data_vio(data_vio); +#endif + u16 size = 0; + int result = -ENOENT; assert_data_vio_on_cpu_thread(data_vio); +#ifdef VDO_COMPRESS_STATS_DEBUG + vdo_log_compress_algorithm_once(vdo); +#endif /* * By putting the compressed data at the start of the compressed block data field, we won't * need to copy it if this data_vio becomes a compressed write agent. */ - size = LZ4_compress_default(data_vio->vio.data, - data_vio->compression.block->data, VDO_BLOCK_SIZE, - VDO_MAX_COMPRESSED_FRAGMENT_SIZE, - (char *) vdo_get_work_queue_private_data()); - if ((size > 0) && (size < VDO_COMPRESSED_BLOCK_DATA_SIZE)) { + if (context != NULL) + result = compress_block(context, data_vio->vio.data, + data_vio->compression.block->data, &size); + +#ifdef VDO_COMPRESS_STATS_DEBUG + vdo_record_compress_attempt(vdo, result, size); +#endif + + if (result == 0) { data_vio->compression.size = size; launch_data_vio_packer_callback(data_vio, pack_compressed_data); return; diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h index 25926b6cd98b..4bd694a6255b 100644 --- a/drivers/md/dm-vdo/data-vio.h +++ b/drivers/md/dm-vdo/data-vio.h @@ -657,6 +657,10 @@ void data_vio_allocate_data_block(struct data_vio *data_vio, void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset); +int __must_check make_vdo_compression_context(const char *compressor, + struct vdo_compression_context **context_ptr); +void free_vdo_compression_context(struct vdo_compression_context *context); + int __must_check uncompress_data_vio(struct data_vio *data_vio, enum block_mapping_state mapping_state, char *buffer); diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index dd05691e4097..498519533b8f 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -6,12 +6,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include "admin-state.h" #include "block-map.h" @@ -140,6 +142,9 @@ static const char * const ADMIN_PHASE_NAMES[] = { /* If we bump this, update the arrays below */ #define TABLE_VERSION 4 +#define VDO_DEFAULT_COMPRESSOR "lz4" +static char vdo_compressor[CRYPTO_MAX_ALG_NAME] = VDO_DEFAULT_COMPRESSOR; + /* arrays for handling different table versions */ static const u8 REQUIRED_ARGC[] = { 10, 12, 9, 7, 6 }; /* pool name no longer used. only here for verification of older versions */ @@ -379,6 +384,35 @@ static inline int __must_check parse_bool(const char *bool_str, const char *true return VDO_SUCCESS; } +static int vdo_compressor_param_set(const char *val, const struct kernel_param *kp) +{ + char compressor[CRYPTO_MAX_ALG_NAME]; + char *name; + int result; + + result = strscpy(compressor, val, sizeof(compressor)); + if (result < 0) + return result; + + name = strstrip(compressor); + if (!crypto_has_acomp(name, 0, 0)) { + vdo_log_error("compressor \"%s\" is not available", name); + return -ENOENT; + } + + return param_set_copystring(name, kp); +} + +static const struct kernel_param_ops vdo_compressor_param_ops = { + .set = vdo_compressor_param_set, + .get = param_get_string, +}; + +static struct kparam_string vdo_compressor_param_string = { + .maxlen = sizeof(vdo_compressor), + .string = vdo_compressor, +}; + /** * process_one_thread_config_spec() - Process one component of a thread parameter configuration * string and update the configuration data structure. @@ -758,6 +792,9 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti, config->max_discard_blocks = 1; config->deduplication = true; config->compression = false; + kernel_param_lock(THIS_MODULE); + strscpy(config->compressor, vdo_compressor, sizeof(config->compressor)); + kernel_param_unlock(THIS_MODULE); arg_set.argc = argc; arg_set.argv = argv; @@ -1105,6 +1142,14 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, if ((argc == 1) && (strcasecmp(argv[0], "stats") == 0)) { vdo_write_stats(vdo, result_buffer, maxlen); result = 1; +#ifdef VDO_COMPRESS_STATS_DEBUG + } else if ((argc == 1) && (strcasecmp(argv[0], "compress-stats") == 0)) { + vdo_write_compress_stats(vdo, result_buffer, maxlen); + result = 1; + } else if ((argc == 1) && (strcasecmp(argv[0], "compress-stats-reset") == 0)) { + vdo_reset_compress_attempt_stats(vdo); + result = 0; +#endif } else { result = vdo_status_to_errno(process_vdo_message(vdo, argc, argv)); } @@ -1459,6 +1504,7 @@ static int vdo_initialize(struct dm_target *ti, unsigned int instance, vdo_log_debug("Block map maximum age = %u", config->block_map_maximum_age); vdo_log_debug("Deduplication = %s", (config->deduplication ? "on" : "off")); vdo_log_debug("Compression = %s", (config->compression ? "on" : "off")); + vdo_log_debug("Compressor = %s", config->compressor); vdo = vdo_find_matching(vdo_uses_device, config); if (vdo != NULL) { @@ -1795,6 +1841,11 @@ static int validate_new_device_config(struct device_config *to_validate, return VDO_PARAMETER_MISMATCH; } + if (strcmp(to_validate->compressor, config->compressor) != 0) { + *error_ptr = "Compression algorithm cannot change"; + return VDO_PARAMETER_MISMATCH; + } + if (to_validate->physical_blocks < config->physical_blocks) { *error_ptr = "Removing physical storage from a VDO is not supported"; return VDO_NOT_IMPLEMENTED; @@ -1878,6 +1929,9 @@ static int update_existing_vdo(const char *device_name, struct dm_target *ti, if (result != VDO_SUCCESS) return -EINVAL; + strscpy(config->compressor, vdo->device_config->compressor, + sizeof(config->compressor)); + vdo_log_info("preparing to modify device '%s'", device_name); result = prepare_to_modify(ti, config, vdo); if (result != VDO_SUCCESS) { @@ -2904,6 +2958,9 @@ module_exit(vdo_exit); module_param_named(log_level, vdo_log_level, uint, 0644); MODULE_PARM_DESC(log_level, "Log level for log messages"); +module_param_cb(compressor, &vdo_compressor_param_ops, + &vdo_compressor_param_string, 0644); +MODULE_PARM_DESC(compressor, "Crypto API compression algorithm for new VDO instances"); MODULE_DESCRIPTION(DM_NAME " target for transparent deduplication"); MODULE_AUTHOR("Red Hat, Inc."); diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index 2802cf92922b..bd55074d9c40 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -11,6 +11,8 @@ #include "thread-device.h" #include "vdo.h" +#include + static void write_u64(char *prefix, u64 value, char *suffix, char **buf, unsigned int *maxlen) { @@ -430,3 +432,38 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) vdo_free(stats); return VDO_SUCCESS; } + +#ifdef VDO_COMPRESS_STATS_DEBUG +static void write_ratio_x100(char *prefix, u64 ratio_x100, char *suffix, + char **buf, unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%llu.%02llux%s", + prefix == NULL ? "" : prefix, + ratio_x100 / 100, ratio_x100 % 100, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +int vdo_write_compress_stats(struct vdo *vdo, char *buf, unsigned int maxlen) +{ + u64 calls = atomic64_read(&vdo->stats.compress_stats_calls); + u64 input_bytes = atomic64_read(&vdo->stats.compress_stats_input_bytes); + u64 payload_bytes = atomic64_read(&vdo->stats.compress_stats_payload_bytes); + u64 saved_bytes = input_bytes > payload_bytes ? input_bytes - payload_bytes : 0; + u64 ratio_x100 = 0; + + if (input_bytes) + ratio_x100 = mul_u64_u64_div_u64(payload_bytes, 100, input_bytes); + + write_u64("calls: ", calls, "\n", &buf, &maxlen); + write_u64("input_bytes: ", input_bytes, "\n", &buf, &maxlen); + write_u64("payload_bytes: ", payload_bytes, "\n", &buf, &maxlen); + write_u64("saved_bytes: ", saved_bytes, "\n", &buf, &maxlen); + write_u64("ratio_x100: ", ratio_x100, "\n", &buf, &maxlen); + write_ratio_x100("ratio: ", ratio_x100, "\n", &buf, &maxlen); + return VDO_SUCCESS; +} +#endif diff --git a/drivers/md/dm-vdo/message-stats.h b/drivers/md/dm-vdo/message-stats.h index f7fceca9acab..0bbd59f1b949 100644 --- a/drivers/md/dm-vdo/message-stats.h +++ b/drivers/md/dm-vdo/message-stats.h @@ -9,5 +9,8 @@ #include "types.h" int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen); +#ifdef VDO_COMPRESS_STATS_DEBUG +int vdo_write_compress_stats(struct vdo *vdo, char *buf, unsigned int maxlen); +#endif #endif /* VDO_MESSAGE_STATS_H */ diff --git a/drivers/md/dm-vdo/types.h b/drivers/md/dm-vdo/types.h index dbe892b10f26..3935808db0b4 100644 --- a/drivers/md/dm-vdo/types.h +++ b/drivers/md/dm-vdo/types.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -225,6 +226,7 @@ struct device_config { unsigned int block_map_maximum_age; bool deduplication; bool compression; + char compressor[CRYPTO_MAX_ALG_NAME]; struct thread_count_config thread_counts; block_count_t max_discard_blocks; }; diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index fff847767755..75d7373b1f24 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include @@ -499,18 +498,19 @@ static int initialize_vdo(struct vdo *vdo, struct device_config *config, config->thread_counts.hash_zones, vdo->thread_config.thread_count); /* Compression context storage */ - result = vdo_allocate(config->thread_counts.cpu_threads, char *, "LZ4 context", - &vdo->compression_context); + result = vdo_allocate(config->thread_counts.cpu_threads, + struct vdo_compression_context *, + "compression context", &vdo->compression_contexts); if (result != VDO_SUCCESS) { - *reason = "cannot allocate LZ4 context"; + *reason = "cannot allocate compression contexts"; return result; } for (i = 0; i < config->thread_counts.cpu_threads; i++) { - result = vdo_allocate(LZ4_MEM_COMPRESS, char, "LZ4 context", - &vdo->compression_context[i]); + result = make_vdo_compression_context(config->compressor, + &vdo->compression_contexts[i]); if (result != VDO_SUCCESS) { - *reason = "cannot allocate LZ4 context"; + *reason = "cannot allocate compression context"; return result; } } @@ -618,7 +618,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason, result = vdo_make_thread(vdo, vdo->thread_config.cpu_thread, &cpu_q_type, config->thread_counts.cpu_threads, - (void **) vdo->compression_context); + (void **) vdo->compression_contexts); if (result != VDO_SUCCESS) { *reason = "CPU queue initialization failed"; return result; @@ -718,11 +718,11 @@ void vdo_destroy(struct vdo *vdo) uninitialize_thread_config(&vdo->thread_config); - if (vdo->compression_context != NULL) { + if (vdo->compression_contexts != NULL) { for (i = 0; i < vdo->device_config->thread_counts.cpu_threads; i++) - vdo_free(vdo_forget(vdo->compression_context[i])); + free_vdo_compression_context(vdo_forget(vdo->compression_contexts[i])); - vdo_free(vdo_forget(vdo->compression_context)); + vdo_free(vdo_forget(vdo->compression_contexts)); } vdo_free(vdo); } @@ -1413,6 +1413,34 @@ bool vdo_get_compressing(struct vdo *vdo) return READ_ONCE(vdo->compressing); } +#ifdef VDO_COMPRESS_STATS_DEBUG +void vdo_log_compress_algorithm_once(struct vdo *vdo) +{ + if (atomic_cmpxchg(&vdo->compression_algorithm_logged, 0, 1) != 0) + return; + + vdo_log_info("VDO compression statistics using algorithm: %s", + vdo->device_config->compressor); +} + +void vdo_record_compress_attempt(struct vdo *vdo, int result, u16 size) +{ + u64 payload_size = result == VDO_SUCCESS ? size : VDO_BLOCK_SIZE; + + atomic64_inc(&vdo->stats.compress_stats_calls); + atomic64_add(VDO_BLOCK_SIZE, &vdo->stats.compress_stats_input_bytes); + atomic64_add(payload_size, &vdo->stats.compress_stats_payload_bytes); +} + +void vdo_reset_compress_attempt_stats(struct vdo *vdo) +{ + atomic64_set(&vdo->stats.compress_stats_calls, 0); + atomic64_set(&vdo->stats.compress_stats_input_bytes, 0); + atomic64_set(&vdo->stats.compress_stats_payload_bytes, 0); + atomic_set(&vdo->compression_algorithm_logged, 0); +} +#endif + static size_t get_block_map_cache_size(const struct vdo *vdo) { return ((size_t) vdo->device_config->cache_size) * VDO_BLOCK_SIZE; diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h index 483ae873e002..c5c94b737255 100644 --- a/drivers/md/dm-vdo/vdo.h +++ b/drivers/md/dm-vdo/vdo.h @@ -22,6 +22,8 @@ #include "thread-registry.h" #include "types.h" +struct vdo_compression_context; + enum notifier_state { /* Notifications are allowed but not in progress */ MAY_NOTIFY, @@ -103,6 +105,11 @@ struct atomic_statistics { struct atomic_bio_stats bios_journal_completed; struct atomic_bio_stats bios_page_cache; struct atomic_bio_stats bios_page_cache_completed; +#ifdef VDO_COMPRESS_STATS_DEBUG + atomic64_t compress_stats_calls; + atomic64_t compress_stats_input_bytes; + atomic64_t compress_stats_payload_bytes; +#endif }; struct read_only_notifier { @@ -207,6 +214,10 @@ struct vdo { struct packer *packer; /* Whether incoming data should be compressed */ bool compressing; +#ifdef VDO_COMPRESS_STATS_DEBUG + /* Whether the configured compression algorithm has been logged for this stats interval. */ + atomic_t compression_algorithm_logged; +#endif /* The handler for flush requests */ struct flusher *flusher; @@ -258,8 +269,8 @@ struct vdo { u64 starting_sector_offset; struct volume_geometry geometry; - /* N blobs of context data for LZ4 code, one per CPU thread. */ - char **compression_context; + /* Compression contexts, one per CPU thread. */ + struct vdo_compression_context **compression_contexts; }; /** @@ -318,6 +329,12 @@ bool vdo_get_compressing(struct vdo *vdo); void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats); +#ifdef VDO_COMPRESS_STATS_DEBUG +void vdo_log_compress_algorithm_once(struct vdo *vdo); +void vdo_record_compress_attempt(struct vdo *vdo, int result, u16 size); +void vdo_reset_compress_attempt_stats(struct vdo *vdo); +#endif + thread_id_t vdo_get_callback_thread_id(void); enum vdo_state __must_check vdo_get_state(const struct vdo *vdo);