Module Name: src Committed By: riastradh Date: Sat Jul 25 22:31:04 UTC 2020
Modified Files: src/sys/crypto/aes/arch/x86: aes_ssse3.h aes_ssse3_impl.c aes_ssse3_subr.c immintrin.h Log Message: Implement AES-CCM with SSSE3. To generate a diff of this commit: cvs rdiff -u -r1.2 -r1.3 src/sys/crypto/aes/arch/x86/aes_ssse3.h \ src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c \ src/sys/crypto/aes/arch/x86/immintrin.h cvs rdiff -u -r1.3 -r1.4 src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/crypto/aes/arch/x86/aes_ssse3.h diff -u src/sys/crypto/aes/arch/x86/aes_ssse3.h:1.2 src/sys/crypto/aes/arch/x86/aes_ssse3.h:1.3 --- src/sys/crypto/aes/arch/x86/aes_ssse3.h:1.2 Sat Jul 25 22:12:57 2020 +++ src/sys/crypto/aes/arch/x86/aes_ssse3.h Sat Jul 25 22:31:04 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: aes_ssse3.h,v 1.2 2020/07/25 22:12:57 riastradh Exp $ */ +/* $NetBSD: aes_ssse3.h,v 1.3 2020/07/25 22:31:04 riastradh Exp $ */ /*- * Copyright (c) 2020 The NetBSD Foundation, Inc. @@ -59,6 +59,12 @@ void aes_ssse3_xts_enc(const struct aese uint8_t[static 16], size_t, uint8_t[static 16], uint32_t); void aes_ssse3_xts_dec(const struct aesdec *, const uint8_t[static 16], uint8_t[static 16], size_t, uint8_t[static 16], uint32_t); +void aes_ssse3_cbcmac_update1(const struct aesenc *, const uint8_t[static 16], + size_t, uint8_t[static 16], uint32_t); +void aes_ssse3_ccm_enc1(const struct aesenc *, const uint8_t[static 16], + uint8_t[static 16], size_t, uint8_t[static 32], uint32_t); +void aes_ssse3_ccm_dec1(const struct aesenc *, const uint8_t[static 16], + uint8_t[static 16], size_t, uint8_t[static 32], uint32_t); int aes_ssse3_selftest(void); Index: src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c diff -u src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c:1.2 src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c:1.3 --- src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c:1.2 Tue Jun 30 20:32:11 2020 +++ src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c Sat Jul 25 22:31:04 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: aes_ssse3_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $ */ +/* $NetBSD: aes_ssse3_subr.c,v 1.3 2020/07/25 22:31:04 riastradh Exp $ */ /*- * Copyright (c) 2020 The NetBSD Foundation, Inc. @@ -27,7 +27,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $"); +__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_subr.c,v 1.3 2020/07/25 22:31:04 riastradh Exp $"); #ifdef _KERNEL #include <sys/systm.h> @@ -208,6 +208,75 @@ aes_ssse3_xts_dec(const struct aesdec *d storeblock(tweak, t); } +void +aes_ssse3_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16], + size_t nbytes, uint8_t auth0[static 16], uint32_t nrounds) +{ + __m128i auth; + + KASSERT(nbytes); + KASSERT(nbytes % 16 == 0); + + auth = loadblock(auth0); + for (; nbytes; nbytes -= 16, in += 16) + auth = aes_ssse3_enc1(enc, auth ^ loadblock(in), nrounds); + storeblock(auth0, auth); +} + +void +aes_ssse3_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32], + uint32_t nrounds) +{ + const __m128i ctr32_inc = _mm_set_epi32(1, 0, 0, 0); + const __m128i bs32 = + _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203); + __m128i auth, ctr_be, ctr, ptxt; + + KASSERT(nbytes); + KASSERT(nbytes % 16 == 0); + + auth = loadblock(authctr); + ctr_be = loadblock(authctr + 16); + ctr = _mm_shuffle_epi8(ctr_be, bs32); + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + ptxt = loadblock(in); + auth = aes_ssse3_enc1(enc, auth ^ ptxt, nrounds); + ctr = _mm_add_epi32(ctr, ctr32_inc); + ctr_be = _mm_shuffle_epi8(ctr, bs32); + storeblock(out, ptxt ^ aes_ssse3_enc1(enc, ctr_be, nrounds)); + } + storeblock(authctr, auth); + storeblock(authctr + 16, ctr_be); +} + +void +aes_ssse3_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32], + uint32_t nrounds) +{ + const __m128i ctr32_inc = _mm_set_epi32(1, 0, 0, 0); + const __m128i bs32 = + _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203); + __m128i auth, ctr_be, ctr, ptxt; + + KASSERT(nbytes); + KASSERT(nbytes % 16 == 0); + + auth = loadblock(authctr); + ctr_be = loadblock(authctr + 16); + ctr = _mm_shuffle_epi8(ctr_be, bs32); + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + ctr = _mm_add_epi32(ctr, ctr32_inc); + ctr_be = _mm_shuffle_epi8(ctr, bs32); + ptxt = loadblock(in) ^ aes_ssse3_enc1(enc, ctr_be, nrounds); + storeblock(out, ptxt); + auth = aes_ssse3_enc1(enc, auth ^ ptxt, nrounds); + } + storeblock(authctr, auth); + storeblock(authctr + 16, ctr_be); +} + int aes_ssse3_selftest(void) { Index: src/sys/crypto/aes/arch/x86/immintrin.h diff -u src/sys/crypto/aes/arch/x86/immintrin.h:1.2 src/sys/crypto/aes/arch/x86/immintrin.h:1.3 --- src/sys/crypto/aes/arch/x86/immintrin.h:1.2 Mon Jun 29 23:51:35 2020 +++ src/sys/crypto/aes/arch/x86/immintrin.h Sat Jul 25 22:31:04 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: immintrin.h,v 1.2 2020/06/29 23:51:35 riastradh Exp $ */ +/* $NetBSD: immintrin.h,v 1.3 2020/07/25 22:31:04 riastradh Exp $ */ /*- * Copyright (c) 2020 The NetBSD Foundation, Inc. @@ -51,6 +51,7 @@ typedef long long __m128i_u typedef long long __v2di __attribute__((__vector_size__(16))); typedef unsigned long long __v2du __attribute__((__vector_size__(16))); typedef int __v4si __attribute__((__vector_size__(16))); +typedef unsigned __v4su __attribute__((__vector_size__(16))); typedef float __v4sf __attribute__((__vector_size__(16))); typedef short __v8hi __attribute__((__vector_size__(16))); typedef char __v16qi __attribute__((__vector_size__(16))); @@ -65,6 +66,7 @@ typedef long long __m128i_u typedef long long __v2di __attribute__((__vector_size__(16))); typedef unsigned long long __v2du __attribute__((__vector_size__(16))); typedef int __v4si __attribute__((__vector_size__(16))); +typedef unsigned __v4su __attribute__((__vector_size__(16))); typedef float __v4sf __attribute__((__vector_size__(16))); typedef short __v8hi __attribute__((__vector_size__(16))); typedef char __v16qi __attribute__((__vector_size__(16))); @@ -83,6 +85,13 @@ typedef char __v16qi __attribute__((__ve #define _SSSE3_ATTR __attribute__((target("ssse3"))) +_INTRINSATTR +static __inline __m128i +_mm_add_epi32(__m128i __a, __m128i __b) +{ + return (__m128i)((__v4su)__a + (__v4su)__b); +} + #if defined(__GNUC__) && !defined(__clang__) #define _mm_alignr_epi8(hi,lo,bytes) \ (__m128i)__builtin_ia32_palignr128((__v2di)(__m128i)(hi), \ Index: src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c diff -u src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c:1.3 src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c:1.4 --- src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c:1.3 Sat Jul 25 22:12:57 2020 +++ src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c Sat Jul 25 22:31:04 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: aes_ssse3_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $ */ +/* $NetBSD: aes_ssse3_impl.c,v 1.4 2020/07/25 22:31:04 riastradh Exp $ */ /*- * Copyright (c) 2020 The NetBSD Foundation, Inc. @@ -27,7 +27,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $"); +__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_impl.c,v 1.4 2020/07/25 22:31:04 riastradh Exp $"); #include <crypto/aes/aes.h> #include <crypto/aes/aes_impl.h> @@ -136,6 +136,39 @@ aes_ssse3_xts_dec_impl(const struct aesd fpu_kern_leave(); } +static void +aes_ssse3_cbcmac_update1_impl(const struct aesenc *enc, + const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16], + uint32_t nrounds) +{ + + fpu_kern_enter(); + aes_ssse3_cbcmac_update1(enc, in, nbytes, auth, nrounds); + fpu_kern_leave(); +} + +static void +aes_ssse3_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32], + uint32_t nrounds) +{ + + fpu_kern_enter(); + aes_ssse3_ccm_enc1(enc, in, out, nbytes, authctr, nrounds); + fpu_kern_leave(); +} + +static void +aes_ssse3_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32], + uint32_t nrounds) +{ + + fpu_kern_enter(); + aes_ssse3_ccm_dec1(enc, in, out, nbytes, authctr, nrounds); + fpu_kern_leave(); +} + static int aes_ssse3_probe(void) { @@ -183,4 +216,7 @@ struct aes_impl aes_ssse3_impl = { .ai_cbc_dec = aes_ssse3_cbc_dec_impl, .ai_xts_enc = aes_ssse3_xts_enc_impl, .ai_xts_dec = aes_ssse3_xts_dec_impl, + .ai_cbcmac_update1 = aes_ssse3_cbcmac_update1_impl, + .ai_ccm_enc1 = aes_ssse3_ccm_enc1_impl, + .ai_ccm_dec1 = aes_ssse3_ccm_dec1_impl, };