Add a dun(...) skcipher template that wraps an inner skcipher whose IV is a wide data-unit-number counter (e.g. dun(xts(aes),le)). When the caller sets skcipher_request::data_unit_size, the template splits the request into cryptlen / data_unit_size sub-requests on the inner cipher, walking the IV +1 per unit. Each inner ->encrypt/->decrypt is a direct call, so only the outer dispatch into the crypto API is indirect -- the per-unit work is not.
The second template parameter selects the counter endianness: dun(...,le) for a little-endian counter (dm-crypt plain64, blk-crypto inline encryption) and dun(...,be) for a big-endian one (dm-crypt plain64be). Those are the only two ways a per-unit IV relates to its neighbour by a +1 step; IV modes that are not such a counter are simply not wrapped. Like cryptd() and pcrypt(), dun() wraps an inner skcipher and changes only how the request is dispatched -- here, split across data units -- performing no cipher transform of its own. A dun() tfm exists solely for multi-DU dispatch, so a request with data_unit_size 0 is rejected with -EINVAL; a caller wanting plain single-DU encryption uses the inner skcipher. A hardware engine that consumes a whole multi-DU request in one pass registers its own dun(...) at a higher cra_priority and is selected automatically by the existing priority mechanism; no per-algorithm capability flag is needed. The generic template is sync-only (the split loop treats any non-zero inner return as terminal), so it resolves against a sync inner cipher (mask | CRYPTO_ALG_ASYNC); async is left to such native drivers. The inner IV must be a whole number of 64-bit limbs and no wider than 32 bytes: 16 covers xts(...), 32 covers the widest inline-encryption mode (Adiantum). Suggested-by: Herbert Xu <[email protected]> Signed-off-by: Leonid Ravich <[email protected]> --- crypto/Kconfig | 14 ++ crypto/Makefile | 1 + crypto/dun.c | 359 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 374 insertions(+) create mode 100644 crypto/dun.c diff --git a/crypto/Kconfig b/crypto/Kconfig index 103d1f58cb7c..4f90a780c4fc 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -746,6 +746,20 @@ config CRYPTO_XTS implementation currently can't handle a sectorsize which is not a multiple of 16 bytes. +config CRYPTO_DUN + tristate "Data-unit-number (DUN) dispatch template" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + help + dun(...) wraps an skcipher whose IV is a wide data-unit-number + counter (e.g. xts(aes)) and lets a caller submit several data units + sharing one starting IV in a single request, via + skcipher_request::data_unit_size. The counter endianness is the + second parameter: dun(xts(aes),le) or dun(xts(aes),be). The template + splits the request into one inner call per data unit; a hardware + driver may register a higher-priority dun(...) that handles the whole + request in one pass. The first user is dm-crypt. + endmenu menu "AEAD (authenticated encryption with associated data) ciphers" diff --git a/crypto/Makefile b/crypto/Makefile index 162242593c7c..584d9e8c4347 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -93,6 +93,7 @@ obj-$(CONFIG_CRYPTO_PCBC) += pcbc.o obj-$(CONFIG_CRYPTO_CTS) += cts.o obj-$(CONFIG_CRYPTO_LRW) += lrw.o obj-$(CONFIG_CRYPTO_XTS) += xts.o +obj-$(CONFIG_CRYPTO_DUN) += dun.o obj-$(CONFIG_CRYPTO_CTR) += ctr.o obj-$(CONFIG_CRYPTO_XCTR) += xctr.o obj-$(CONFIG_CRYPTO_HCTR2) += hctr2.o diff --git a/crypto/dun.c b/crypto/dun.c new file mode 100644 index 000000000000..4fcb81a025b9 --- /dev/null +++ b/crypto/dun.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dun: data-unit-number dispatch template for skcipher + * + * Wraps an inner skcipher (e.g. xts(aes)) and, when the caller sets + * skcipher_request::data_unit_size, splits the request into cryptlen / + * data_unit_size sub-requests, each unit's IV the previous one +1 -- the + * data-unit-number (DUN) convention. The second parameter selects the IV + * walk (see struct dun_mode): dun(xts(aes),le) or dun(xts(aes),be). + * + * Like cryptd()/pcrypt(), dun() only changes how a request is dispatched and + * performs no transform of its own; a native one-pass multi-DU driver wins by + * cra_priority. Callers that never set data_unit_size pay nothing. + */ + +#include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> + +/* Bounds the on-stack IV buffers: 16 covers xts(...), 32 covers Adiantum. */ +#define DUN_MAX_IVSIZE 32 + +/* + * A dun() mode is the rule for deriving each data unit's IV from the request's + * starting IV. @name is the template's second parameter; @iv_next advances the + * @ivsize-byte @iv in place to the next data unit. @ivsize_ok rejects IV sizes + * the walk can't handle. Add a row to dun_modes[] to support a new convention. + */ +struct dun_mode { + const char *name; + void (*iv_next)(u8 *iv, unsigned int ivsize); + bool (*ivsize_ok)(unsigned int ivsize); +}; + +struct dun_tfm_ctx { + struct crypto_skcipher *child; + const struct dun_mode *mode; +}; + +struct dun_inst_ctx { + struct crypto_skcipher_spawn spawn; + const struct dun_mode *mode; +}; + +struct dun_request_ctx { + /* Must be last; the child request is appended with its own reqsize. */ + struct skcipher_request subreq; +}; + +/* Little-endian counter: increment the IV per __le64 limb, low limb first. */ +static void dun_iv_next_le(u8 *iv, unsigned int ivsize) +{ + unsigned int i; + + for (i = 0; i < ivsize; i += sizeof(__le64)) { + __le64 limb; + u64 v; + + memcpy(&limb, iv + i, sizeof(limb)); + v = le64_to_cpu(limb) + 1; + limb = cpu_to_le64(v); + memcpy(iv + i, &limb, sizeof(limb)); + if (likely(v != 0)) + break; /* no carry into the next limb */ + } +} + +/* Big-endian counter: increment the IV byte-wise from the last byte. */ +static void dun_iv_next_be(u8 *iv, unsigned int ivsize) +{ + unsigned int i = ivsize; + + while (i--) { + if (likely(++iv[i])) + break; /* no carry into the next byte */ + } +} + +/* + * le requires this: it walks the IV in __le64 limbs, so the size must be a + * whole number of limbs. be increments byte-wise and would accept any size, + * but reuses the same check for a uniform value-space. + */ +static bool dun_ivsize_whole_limbs(unsigned int ivsize) +{ + return IS_ALIGNED(ivsize, sizeof(__le64)); +} + +static const struct dun_mode dun_modes[] = { + { "le", dun_iv_next_le, dun_ivsize_whole_limbs }, + { "be", dun_iv_next_be, dun_ivsize_whole_limbs }, +}; + +static const struct dun_mode *dun_find_mode(const char *name) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(dun_modes); i++) + if (!strcmp(name, dun_modes[i].name)) + return &dun_modes[i]; + return NULL; +} + +static int dun_setkey(struct crypto_skcipher *parent, const u8 *key, + unsigned int keylen) +{ + struct dun_tfm_ctx *ctx = crypto_skcipher_ctx(parent); + struct crypto_skcipher *child = ctx->child; + + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + return crypto_skcipher_setkey(child, key, keylen); +} + +/* + * Run one inner ->crypt per data unit, walking the IV as a wide counter. + * @req->iv is never modified; the inner cipher only sees the iv_unit copy. + */ +static int dun_split(struct skcipher_request *req, + int (*crypt)(struct skcipher_request *)) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct dun_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dun_request_ctx *rctx = skcipher_request_ctx(req); + struct skcipher_request *subreq = &rctx->subreq; + const unsigned int du = req->data_unit_size; + const unsigned int total = req->cryptlen; + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + const struct dun_mode *mode = ctx->mode; + bool inplace = req->src == req->dst; + struct scatter_walk src_walk, dst_walk; + struct scatterlist src_sg[2], dst_sg[2]; + u8 iv_ctr[DUN_MAX_IVSIZE]; + u8 iv_unit[DUN_MAX_IVSIZE]; + unsigned int off; + int err = 0; + + /* iv_ctr is the counter; iv_unit is a per-unit copy an inner may write + * back in place (e.g. xts, essiv), so the counter is never mutated. + */ + memcpy(iv_ctr, req->iv, ivsize); + + sg_init_table(src_sg, 2); + scatterwalk_start(&src_walk, req->src); + if (!inplace) { + sg_init_table(dst_sg, 2); + scatterwalk_start(&dst_walk, req->dst); + } + + skcipher_request_set_tfm(subreq, ctx->child); + skcipher_request_set_callback(subreq, skcipher_request_flags(req), + NULL, NULL); + + for (off = 0; off < total; off += du) { + struct scatterlist *s, *d; + + scatterwalk_get_sglist(&src_walk, src_sg); + scatterwalk_skip(&src_walk, du); + s = src_sg; + if (inplace) { + d = src_sg; + } else { + scatterwalk_get_sglist(&dst_walk, dst_sg); + scatterwalk_skip(&dst_walk, du); + d = dst_sg; + } + + memcpy(iv_unit, iv_ctr, ivsize); + skcipher_request_set_crypt(subreq, s, d, du, iv_unit); + err = crypt(subreq); + if (err) + break; + + mode->iv_next(iv_ctr, ivsize); + } + + return err; +} + +/* + * Validate a multi-DU request: non-zero cryptlen, and a data_unit_size that is + * set, a multiple of the block size, and divides cryptlen evenly. + */ +static int dun_check(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + + if (!req->cryptlen || !req->data_unit_size || + !IS_ALIGNED(req->data_unit_size, crypto_skcipher_blocksize(tfm)) || + (req->cryptlen % req->data_unit_size)) + return -EINVAL; + return 0; +} + +static int dun_encrypt(struct skcipher_request *req) +{ + int err = dun_check(req); + + if (err) + return err; + return dun_split(req, crypto_skcipher_encrypt); +} + +static int dun_decrypt(struct skcipher_request *req) +{ + int err = dun_check(req); + + if (err) + return err; + return dun_split(req, crypto_skcipher_decrypt); +} + +static int dun_init_tfm(struct crypto_skcipher *tfm) +{ + struct skcipher_instance *inst = skcipher_alg_instance(tfm); + struct dun_inst_ctx *ictx = skcipher_instance_ctx(inst); + struct dun_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child; + + child = crypto_spawn_skcipher(&ictx->spawn); + if (IS_ERR(child)) + return PTR_ERR(child); + + ctx->child = child; + ctx->mode = ictx->mode; + crypto_skcipher_set_reqsize(tfm, + sizeof(struct dun_request_ctx) + + crypto_skcipher_reqsize(child)); + return 0; +} + +static void dun_exit_tfm(struct crypto_skcipher *tfm) +{ + struct dun_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->child); +} + +static void dun_free_instance(struct skcipher_instance *inst) +{ + struct dun_inst_ctx *ictx = skcipher_instance_ctx(inst); + + crypto_drop_skcipher(&ictx->spawn); + kfree(inst); +} + +static int dun_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct skcipher_alg_common *alg; + struct skcipher_instance *inst; + struct dun_inst_ctx *ictx; + const struct dun_mode *mode; + const char *cipher_name; + const char *mode_name; + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask); + if (err) + return err; + + cipher_name = crypto_attr_alg_name(tb[1]); + if (IS_ERR(cipher_name)) + return PTR_ERR(cipher_name); + + /* Second parameter: the IV-walk mode (see dun_modes[]). */ + mode_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(mode_name)) + return PTR_ERR(mode_name); + mode = dun_find_mode(mode_name); + if (!mode) + return -EINVAL; + + inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); + if (!inst) + return -ENOMEM; + ictx = skcipher_instance_ctx(inst); + ictx->mode = mode; + + /* + * Sync-only: the split loop can't drive an async (-EINPROGRESS) child, + * so resolve against a sync inner (mask | CRYPTO_ALG_ASYNC). + */ + err = crypto_grab_skcipher(&ictx->spawn, skcipher_crypto_instance(inst), + cipher_name, 0, mask | CRYPTO_ALG_ASYNC); + if (err) + goto err_free_inst; + + alg = crypto_spawn_skcipher_alg_common(&ictx->spawn); + + /* The mode must accept this IV size, and it must fit our buffers. */ + err = -EINVAL; + if (!alg->ivsize || alg->ivsize > DUN_MAX_IVSIZE || + !mode->ivsize_ok(alg->ivsize)) + goto err_free_inst; + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "dun(%s,%s)", + alg->base.cra_name, mode->name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "dun(%s,%s)", alg->base.cra_driver_name, + mode->name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + inst->alg.base.cra_priority = alg->base.cra_priority; + inst->alg.base.cra_blocksize = alg->base.cra_blocksize; + inst->alg.base.cra_alignmask = alg->base.cra_alignmask; + inst->alg.base.cra_ctxsize = sizeof(struct dun_tfm_ctx); + + inst->alg.ivsize = alg->ivsize; + inst->alg.chunksize = alg->chunksize; + inst->alg.min_keysize = alg->min_keysize; + inst->alg.max_keysize = alg->max_keysize; + + inst->alg.init = dun_init_tfm; + inst->alg.exit = dun_exit_tfm; + inst->alg.setkey = dun_setkey; + inst->alg.encrypt = dun_encrypt; + inst->alg.decrypt = dun_decrypt; + + inst->free = dun_free_instance; + + err = skcipher_register_instance(tmpl, inst); + if (err) { +err_free_inst: + dun_free_instance(inst); + } + return err; +} + +static struct crypto_template dun_tmpl = { + .name = "dun", + .create = dun_create, + .module = THIS_MODULE, +}; + +static int __init dun_module_init(void) +{ + return crypto_register_template(&dun_tmpl); +} + +static void __exit dun_module_exit(void) +{ + crypto_unregister_template(&dun_tmpl); +} + +module_init(dun_module_init); +module_exit(dun_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Data-unit-number dispatch template for skcipher"); +MODULE_ALIAS_CRYPTO("dun"); -- 2.47.3
