arch/arm/crypto/ghash-ce-glue.c originally provided only a "ghash" crypto_shash algorithm using PMULL if available, else NEON.
Significantly later, it was updated to also provide a full AES-GCM implementation using PMULL. This made the PMULL support in the "ghash" crypto_shash largely obsolete. Indeed, the arm64 equivalent of this file unconditionally uses only ASIMD in its "ghash" crypto_shash. Given that inconsistency and the fact that the NEON-only code is more easily separable into the GHASH library than the PMULL based code is, let's align with arm64 and just support NEON-only for the pure GHASH. Signed-off-by: Eric Biggers <[email protected]> --- arch/arm/crypto/ghash-ce-glue.c | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 454adcc62cc6..d7d787de7dd3 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -34,11 +34,11 @@ MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); #define RFC4106_NONCE_SIZE 4 struct ghash_key { be128 k; - u64 h[][2]; + u64 h[1][2]; }; struct gcm_key { u64 h[4][2]; u32 rk[AES_MAX_KEYLENGTH_U32]; @@ -49,16 +49,14 @@ struct gcm_key { struct arm_ghash_desc_ctx { u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; }; asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); + u64 const h[4][2], const char *head); asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); + u64 const h[1][2], const char *head); static int ghash_init(struct shash_desc *desc) { struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); @@ -68,14 +66,11 @@ static int ghash_init(struct shash_desc *desc) static void ghash_do_update(int blocks, u64 dg[], const char *src, struct ghash_key *key, const char *head) { kernel_neon_begin(); - if (static_branch_likely(&use_p64)) - pmull_ghash_update_p64(blocks, dg, src, key->h, head); - else - pmull_ghash_update_p8(blocks, dg, src, key->h, head); + pmull_ghash_update_p8(blocks, dg, src, key->h, head); kernel_neon_end(); } static int ghash_update(struct shash_desc *desc, const u8 *src, unsigned int len) @@ -145,23 +140,10 @@ static int ghash_setkey(struct crypto_shash *tfm, return -EINVAL; /* needed for the fallback */ memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); ghash_reflect(key->h[0], &key->k); - - if (static_branch_likely(&use_p64)) { - be128 h = key->k; - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[1], &h); - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[2], &h); - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[3], &h); - } return 0; } static struct shash_alg ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, @@ -173,15 +155,15 @@ static struct shash_alg ghash_alg = { .import = ghash_import, .descsize = sizeof(struct arm_ghash_desc_ctx), .statesize = sizeof(struct ghash_desc_ctx), .base.cra_name = "ghash", - .base.cra_driver_name = "ghash-ce", + .base.cra_driver_name = "ghash-neon", .base.cra_priority = 300, .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = GHASH_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), + .base.cra_ctxsize = sizeof(struct ghash_key), .base.cra_module = THIS_MODULE, }; void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, struct gcm_key const *k, char *dst, @@ -569,12 +551,10 @@ static int __init ghash_ce_mod_init(void) if (elf_hwcap2 & HWCAP2_PMULL) { err = crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); if (err) return err; - ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]); - static_branch_enable(&use_p64); } err = crypto_register_shash(&ghash_alg); if (err) goto err_aead; -- 2.53.0
