Module Name: src
Committed By: riastradh
Date: Sat Jul 25 22:31:32 UTC 2020
Modified Files:
src/sys/crypto/aes/arch/x86: aes_via.c
Log Message:
Implement AES-CCM with VIA ACE.
To generate a diff of this commit:
cvs rdiff -u -r1.4 -r1.5 src/sys/crypto/aes/arch/x86/aes_via.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/crypto/aes/arch/x86/aes_via.c
diff -u src/sys/crypto/aes/arch/x86/aes_via.c:1.4 src/sys/crypto/aes/arch/x86/aes_via.c:1.5
--- src/sys/crypto/aes/arch/x86/aes_via.c:1.4 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_via.c Sat Jul 25 22:31:32 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_via.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_via.c,v 1.5 2020/07/25 22:31:32 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.5 2020/07/25 22:31:32 riastradh Exp $");
#ifdef _KERNEL
#include <sys/types.h>
@@ -674,6 +674,176 @@ aesvia_xts_dec(const struct aesdec *dec,
explicit_memset(t, 0, sizeof t);
}
+static struct evcnt cbcmac_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "cbcmac aligned");
+EVCNT_ATTACH_STATIC(cbcmac_aligned_evcnt);
+static struct evcnt cbcmac_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "cbcmac unaligned");
+EVCNT_ATTACH_STATIC(cbcmac_unaligned_evcnt);
+
+static void
+aesvia_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
+ size_t nbytes, uint8_t auth0[static 16], uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+ uint8_t authbuf[16] __aligned(16);
+ uint8_t *auth = auth0;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ if ((uintptr_t)auth0 & 0xf) {
+ memcpy(authbuf, auth0, 16);
+ auth = authbuf;
+ cbcmac_unaligned_evcnt.ev_count++;
+ } else {
+ cbcmac_aligned_evcnt.ev_count++;
+ }
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ for (; nbytes; nbytes -= 16, in += 16) {
+ xor128(auth, auth, in);
+ aesvia_encN(enc, auth, auth, 1, cw0);
+ }
+ fpu_kern_leave();
+
+ if ((uintptr_t)auth0 & 0xf) {
+ memcpy(auth0, authbuf, 16);
+ explicit_memset(authbuf, 0, sizeof authbuf);
+ }
+}
+
+static struct evcnt ccmenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "ccmenc aligned");
+EVCNT_ATTACH_STATIC(ccmenc_aligned_evcnt);
+static struct evcnt ccmenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "ccmenc unaligned");
+EVCNT_ATTACH_STATIC(ccmenc_unaligned_evcnt);
+
+static void
+aesvia_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr0[static 32],
+ uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+ uint8_t authctrbuf[32] __aligned(16);
+ uint8_t *authctr;
+ uint32_t c0, c1, c2, c3;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ if ((uintptr_t)authctr0 & 0xf) {
+ memcpy(authctrbuf, authctr0, 16);
+ authctr = authctrbuf;
+ ccmenc_unaligned_evcnt.ev_count++;
+ } else {
+ ccmenc_aligned_evcnt.ev_count++;
+ }
+ c0 = le32dec(authctr0 + 16 + 4*0);
+ c1 = le32dec(authctr0 + 16 + 4*1);
+ c2 = le32dec(authctr0 + 16 + 4*2);
+ c3 = be32dec(authctr0 + 16 + 4*3);
+
+ /*
+ * In principle we could use REP XCRYPTCTR here, but that
+ * doesn't help to compute the CBC-MAC step, and certain VIA
+ * CPUs have some weird errata with REP XCRYPTCTR that make it
+ * kind of a pain to use. So let's just use REP XCRYPTECB to
+ * simultaneously compute the CBC-MAC step and the CTR step.
+ * (Maybe some VIA CPUs will compute REP XCRYPTECB in parallel,
+ * who knows...)
+ */
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ xor128(authctr, authctr, in);
+ le32enc(authctr + 16 + 4*0, c0);
+ le32enc(authctr + 16 + 4*1, c1);
+ le32enc(authctr + 16 + 4*2, c2);
+ be32enc(authctr + 16 + 4*3, ++c3);
+ aesvia_encN(enc, authctr, authctr, 2, cw0);
+ xor128(out, in, authctr + 16);
+ }
+ fpu_kern_leave();
+
+ if ((uintptr_t)authctr0 & 0xf) {
+ memcpy(authctr0, authctrbuf, 16);
+ explicit_memset(authctrbuf, 0, sizeof authctrbuf);
+ }
+
+ le32enc(authctr0 + 16 + 4*0, c0);
+ le32enc(authctr0 + 16 + 4*1, c1);
+ le32enc(authctr0 + 16 + 4*2, c2);
+ be32enc(authctr0 + 16 + 4*3, c3);
+}
+
+static struct evcnt ccmdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "ccmdec aligned");
+EVCNT_ATTACH_STATIC(ccmdec_aligned_evcnt);
+static struct evcnt ccmdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "ccmdec unaligned");
+EVCNT_ATTACH_STATIC(ccmdec_unaligned_evcnt);
+
+static void
+aesvia_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr0[static 32],
+ uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+ uint8_t authctrbuf[32] __aligned(16);
+ uint8_t *authctr;
+ uint32_t c0, c1, c2, c3;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ c0 = le32dec(authctr0 + 16 + 4*0);
+ c1 = le32dec(authctr0 + 16 + 4*1);
+ c2 = le32dec(authctr0 + 16 + 4*2);
+ c3 = be32dec(authctr0 + 16 + 4*3);
+
+ if ((uintptr_t)authctr0 & 0xf) {
+ memcpy(authctrbuf, authctr0, 16);
+ authctr = authctrbuf;
+ le32enc(authctr + 16 + 4*0, c0);
+ le32enc(authctr + 16 + 4*1, c1);
+ le32enc(authctr + 16 + 4*2, c2);
+ ccmdec_unaligned_evcnt.ev_count++;
+ } else {
+ ccmdec_aligned_evcnt.ev_count++;
+ }
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ be32enc(authctr + 16 + 4*3, ++c3);
+ aesvia_encN(enc, authctr + 16, authctr + 16, 1, cw0);
+ for (;; in += 16, out += 16) {
+ xor128(out, authctr + 16, in);
+ xor128(authctr, authctr, out);
+ if ((nbytes -= 16) == 0)
+ break;
+ le32enc(authctr + 16 + 4*0, c0);
+ le32enc(authctr + 16 + 4*1, c1);
+ le32enc(authctr + 16 + 4*2, c2);
+ be32enc(authctr + 16 + 4*3, ++c3);
+ aesvia_encN(enc, authctr, authctr, 2, cw0);
+ }
+ aesvia_encN(enc, authctr, authctr, 1, cw0);
+ fpu_kern_leave();
+
+ if ((uintptr_t)authctr0 & 0xf) {
+ memcpy(authctr0, authctrbuf, 16);
+ explicit_memset(authctrbuf, 0, sizeof authctrbuf);
+ }
+
+ le32enc(authctr0 + 16 + 4*0, c0);
+ le32enc(authctr0 + 16 + 4*1, c1);
+ le32enc(authctr0 + 16 + 4*2, c2);
+ be32enc(authctr0 + 16 + 4*3, c3);
+}
+
static int
aesvia_probe(void)
{
@@ -727,4 +897,7 @@ struct aes_impl aes_via_impl = {
.ai_cbc_dec = aesvia_cbc_dec,
.ai_xts_enc = aesvia_xts_enc,
.ai_xts_dec = aesvia_xts_dec,
+ .ai_cbcmac_update1 = aesvia_cbcmac_update1,
+ .ai_ccm_enc1 = aesvia_ccm_enc1,
+ .ai_ccm_dec1 = aesvia_ccm_dec1,
};
