Module Name: src
Committed By: riastradh
Date: Sat Jul 25 22:29:56 UTC 2020
Modified Files:
src/sys/crypto/aes/arch/x86: aes_sse2.h aes_sse2_impl.c aes_sse2_subr.c
Log Message:
Implement AES-CCM with SSE2.
To generate a diff of this commit:
cvs rdiff -u -r1.3 -r1.4 src/sys/crypto/aes/arch/x86/aes_sse2.h
cvs rdiff -u -r1.4 -r1.5 src/sys/crypto/aes/arch/x86/aes_sse2_impl.c
cvs rdiff -u -r1.2 -r1.3 src/sys/crypto/aes/arch/x86/aes_sse2_subr.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/crypto/aes/arch/x86/aes_sse2.h
diff -u src/sys/crypto/aes/arch/x86/aes_sse2.h:1.3 src/sys/crypto/aes/arch/x86/aes_sse2.h:1.4
--- src/sys/crypto/aes/arch/x86/aes_sse2.h:1.3 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_sse2.h Sat Jul 25 22:29:56 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_sse2.h,v 1.3 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_sse2.h,v 1.4 2020/07/25 22:29:56 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -58,6 +58,12 @@ void aes_sse2_xts_enc(const struct aesen
uint8_t[static 16], size_t nbytes, uint8_t[static 16], uint32_t);
void aes_sse2_xts_dec(const struct aesdec *, const uint8_t[static 16],
uint8_t[static 16], size_t nbytes, uint8_t[static 16], uint32_t);
+void aes_sse2_cbcmac_update1(const struct aesenc *, const uint8_t[static 16],
+ size_t, uint8_t[static 16], uint32_t);
+void aes_sse2_ccm_enc1(const struct aesenc *, const uint8_t[static 16],
+ uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+void aes_sse2_ccm_dec1(const struct aesenc *, const uint8_t[static 16],
+ uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
int aes_sse2_selftest(void);
Index: src/sys/crypto/aes/arch/x86/aes_sse2_impl.c
diff -u src/sys/crypto/aes/arch/x86/aes_sse2_impl.c:1.4 src/sys/crypto/aes/arch/x86/aes_sse2_impl.c:1.5
--- src/sys/crypto/aes/arch/x86/aes_sse2_impl.c:1.4 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_sse2_impl.c Sat Jul 25 22:29:56 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_sse2_impl.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_sse2_impl.c,v 1.5 2020/07/25 22:29:56 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_sse2_impl.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_sse2_impl.c,v 1.5 2020/07/25 22:29:56 riastradh Exp $");
#include <sys/types.h>
#include <sys/endian.h>
@@ -143,6 +143,39 @@ aes_sse2_xts_dec_impl(const struct aesde
fpu_kern_leave();
}
+static void
+aes_sse2_cbcmac_update1_impl(const struct aesenc *enc,
+ const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
+ uint32_t nrounds)
+{
+
+ fpu_kern_enter();
+ aes_sse2_cbcmac_update1(enc, in, nbytes, auth, nrounds);
+ fpu_kern_leave();
+}
+
+static void
+aes_sse2_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+
+ fpu_kern_enter();
+ aes_sse2_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
+ fpu_kern_leave();
+}
+
+static void
+aes_sse2_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+
+ fpu_kern_enter();
+ aes_sse2_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
+ fpu_kern_leave();
+}
+
static int
aes_sse2_probe(void)
{
@@ -182,4 +215,7 @@ struct aes_impl aes_sse2_impl = {
.ai_cbc_dec = aes_sse2_cbc_dec_impl,
.ai_xts_enc = aes_sse2_xts_enc_impl,
.ai_xts_dec = aes_sse2_xts_dec_impl,
+ .ai_cbcmac_update1 = aes_sse2_cbcmac_update1_impl,
+ .ai_ccm_enc1 = aes_sse2_ccm_enc1_impl,
+ .ai_ccm_dec1 = aes_sse2_ccm_dec1_impl,
};
Index: src/sys/crypto/aes/arch/x86/aes_sse2_subr.c
diff -u src/sys/crypto/aes/arch/x86/aes_sse2_subr.c:1.2 src/sys/crypto/aes/arch/x86/aes_sse2_subr.c:1.3
--- src/sys/crypto/aes/arch/x86/aes_sse2_subr.c:1.2 Tue Jun 30 20:32:11 2020
+++ src/sys/crypto/aes/arch/x86/aes_sse2_subr.c Sat Jul 25 22:29:56 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_sse2_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $ */
+/* $NetBSD: aes_sse2_subr.c,v 1.3 2020/07/25 22:29:56 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_sse2_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_sse2_subr.c,v 1.3 2020/07/25 22:29:56 riastradh Exp $");
#ifdef _KERNEL
#include <sys/systm.h>
@@ -518,6 +518,180 @@ out: /* Store the updated tweak. */
explicit_memset(t, 0, sizeof t);
}
+void
+aes_sse2_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
+ size_t nbytes, uint8_t auth[static 16], uint32_t nrounds)
+{
+ uint64_t sk_exp[120];
+ __m128i q[4];
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ /* Expand round keys for bitslicing. */
+ aes_sse2_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);
+
+ /* Load initial authenticator. */
+ q[0] = aes_sse2_interleave_in(_mm_loadu_epi8(auth));
+
+ for (; nbytes; nbytes -= 16, in += 16) {
+ q[0] ^= aes_sse2_interleave_in(_mm_loadu_epi8(in));
+ aes_sse2_ortho(q);
+ aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+ aes_sse2_ortho(q);
+ }
+
+ /* Store updated authenticator. */
+ _mm_storeu_epi8(auth, aes_sse2_interleave_out(q[0]));
+
+ /* Paranoia: Zero temporary buffers. */
+ explicit_memset(sk_exp, 0, sizeof sk_exp);
+ explicit_memset(q, 0, sizeof q);
+}
+
+void
+aes_sse2_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+ uint64_t sk_exp[120];
+ __m128i q[4];
+ __m128i ctr;
+ uint32_t c0, c1, c2, c3;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ /* Expand round keys for bitslicing. */
+ aes_sse2_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);
+
+ /* Set first block to authenticator. */
+ q[0] = aes_sse2_interleave_in(_mm_loadu_epi8(authctr));
+
+ /* Load initial counter block, big-endian so we can increment it. */
+ c0 = le32dec(authctr + 16 + 4*0);
+ c1 = le32dec(authctr + 16 + 4*1);
+ c2 = le32dec(authctr + 16 + 4*2);
+ c3 = be32dec(authctr + 16 + 4*3);
+
+ /* Set other blocks to garbage -- can't take advantage. */
+ q[2] = q[3] = _mm_setzero_si128();
+
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ /* Update authenticator. */
+ q[0] ^= aes_sse2_interleave_in(_mm_loadu_epi8(in));
+
+ /* Increment 32-bit counter. */
+ ctr = _mm_set_epi32(bswap32(++c3), c2, c1, c0);
+ q[1] = aes_sse2_interleave_in(ctr);
+
+ /* Encrypt authenticator and counter. */
+ aes_sse2_ortho(q);
+ aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+ aes_sse2_ortho(q);
+
+ /* Encrypt with CTR output. */
+ _mm_storeu_epi8(out,
+ _mm_loadu_epi8(in) ^ aes_sse2_interleave_out(q[1]));
+ }
+
+ /* Update authenticator. */
+ _mm_storeu_epi8(authctr, aes_sse2_interleave_out(q[0]));
+
+ /* Update counter. */
+ be32enc(authctr + 16 + 4*3, c3);
+
+ /* Paranoia: Zero temporary buffers. */
+ explicit_memset(sk_exp, 0, sizeof sk_exp);
+ explicit_memset(q, 0, sizeof q);
+}
+
+void
+aes_sse2_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+ uint64_t sk_exp[120];
+ __m128i q[4];
+ __m128i ctr, block;
+ uint32_t c0, c1, c2, c3;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ /* Expand round keys for bitslicing. */
+ aes_sse2_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);
+
+ /* Load initial counter block, big-endian so we can increment it. */
+ c0 = le32dec(authctr + 16 + 4*0);
+ c1 = le32dec(authctr + 16 + 4*1);
+ c2 = le32dec(authctr + 16 + 4*2);
+ c3 = be32dec(authctr + 16 + 4*3);
+
+ /* Increment 32-bit counter. */
+ ctr = _mm_set_epi32(bswap32(++c3), c2, c1, c0);
+ q[0] = aes_sse2_interleave_in(ctr);
+
+ /*
+ * Set the other blocks to garbage -- we don't have any
+ * plaintext to authenticate yet.
+ */
+ q[1] = q[2] = q[3] = _mm_setzero_si128();
+
+ /* Encrypt first CTR. */
+ aes_sse2_ortho(q);
+ aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+ aes_sse2_ortho(q);
+
+ /* Load the initial authenticator. */
+ q[1] = aes_sse2_interleave_in(_mm_loadu_epi8(authctr));
+
+ for (;; in += 16, out += 16) {
+ /* Decrypt the block. */
+ block = _mm_loadu_epi8(in) ^ aes_sse2_interleave_out(q[0]);
+
+ /* Update authenticator. */
+ q[1] ^= aes_sse2_interleave_in(block);
+
+ /* Store plaintext. */
+ _mm_storeu_epi8(out, block);
+
+ /* If this is the last block, stop. */
+ if ((nbytes -= 16) == 0)
+ break;
+
+ /* Increment 32-bit counter. */
+ ctr = _mm_set_epi32(bswap32(++c3), c2, c1, c0);
+ q[0] = aes_sse2_interleave_in(ctr);
+
+ /* Authenticate previous plaintext, encrypt next CTR. */
+ aes_sse2_ortho(q);
+ aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+ aes_sse2_ortho(q);
+ }
+
+ /*
+ * Authenticate last plaintext. We're only doing this for the
+ * authenticator, not for the counter, so don't bother to
+ * initialize q[0], q[2], q[3]. (Even for the sake of
+ * sanitizers, they're already initialized to something by
+ * now.)
+ */
+ aes_sse2_ortho(q);
+ aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+ aes_sse2_ortho(q);
+
+ /* Update authenticator. */
+ _mm_storeu_epi8(authctr, aes_sse2_interleave_out(q[1]));
+
+ /* Update counter. */
+ be32enc(authctr + 16 + 4*3, c3);
+
+ /* Paranoia: Zero temporary buffers. */
+ explicit_memset(sk_exp, 0, sizeof sk_exp);
+ explicit_memset(q, 0, sizeof q);
+}
+
int
aes_sse2_selftest(void)
{
