Module Name:    src
Committed By:   riastradh
Date:           Sat Jul 25 22:31:32 UTC 2020

Modified Files:
        src/sys/crypto/aes/arch/x86: aes_via.c

Log Message:
Implement AES-CCM with VIA ACE.


To generate a diff of this commit:
cvs rdiff -u -r1.4 -r1.5 src/sys/crypto/aes/arch/x86/aes_via.c

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: src/sys/crypto/aes/arch/x86/aes_via.c
diff -u src/sys/crypto/aes/arch/x86/aes_via.c:1.4 src/sys/crypto/aes/arch/x86/aes_via.c:1.5
--- src/sys/crypto/aes/arch/x86/aes_via.c:1.4	Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_via.c	Sat Jul 25 22:31:32 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: aes_via.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $	*/
+/*	$NetBSD: aes_via.c,v 1.5 2020/07/25 22:31:32 riastradh Exp $	*/
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.5 2020/07/25 22:31:32 riastradh Exp $");
 
 #ifdef _KERNEL
 #include <sys/types.h>
@@ -674,6 +674,176 @@ aesvia_xts_dec(const struct aesdec *dec,
 	explicit_memset(t, 0, sizeof t);
 }
 
+static struct evcnt cbcmac_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+    NULL, "aesvia", "cbcmac aligned");
+EVCNT_ATTACH_STATIC(cbcmac_aligned_evcnt);
+static struct evcnt cbcmac_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+    NULL, "aesvia", "cbcmac unaligned");
+EVCNT_ATTACH_STATIC(cbcmac_unaligned_evcnt);
+
+static void
+aesvia_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
+    size_t nbytes, uint8_t auth0[static 16], uint32_t nrounds)
+{
+	const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+	uint8_t authbuf[16] __aligned(16);
+	uint8_t *auth = auth0;
+
+	KASSERT(nbytes);
+	KASSERT(nbytes % 16 == 0);
+
+	if ((uintptr_t)auth0 & 0xf) {
+		memcpy(authbuf, auth0, 16);
+		auth = authbuf;
+		cbcmac_unaligned_evcnt.ev_count++;
+	} else {
+		cbcmac_aligned_evcnt.ev_count++;
+	}
+
+	fpu_kern_enter();
+	aesvia_reload_keys();
+	for (; nbytes; nbytes -= 16, in += 16) {
+		xor128(auth, auth, in);
+		aesvia_encN(enc, auth, auth, 1, cw0);
+	}
+	fpu_kern_leave();
+
+	if ((uintptr_t)auth0 & 0xf) {
+		memcpy(auth0, authbuf, 16);
+		explicit_memset(authbuf, 0, sizeof authbuf);
+	}
+}
+
+static struct evcnt ccmenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+    NULL, "aesvia", "ccmenc aligned");
+EVCNT_ATTACH_STATIC(ccmenc_aligned_evcnt);
+static struct evcnt ccmenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+    NULL, "aesvia", "ccmenc unaligned");
+EVCNT_ATTACH_STATIC(ccmenc_unaligned_evcnt);
+
+static void
+aesvia_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr0[static 32],
+    uint32_t nrounds)
+{
+	const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+	uint8_t authctrbuf[32] __aligned(16);
+	uint8_t *authctr;
+	uint32_t c0, c1, c2, c3;
+
+	KASSERT(nbytes);
+	KASSERT(nbytes % 16 == 0);
+
+	if ((uintptr_t)authctr0 & 0xf) {
+		memcpy(authctrbuf, authctr0, 16);
+		authctr = authctrbuf;
+		ccmenc_unaligned_evcnt.ev_count++;
+	} else {
+		ccmenc_aligned_evcnt.ev_count++;
+	}
+	c0 = le32dec(authctr0 + 16 + 4*0);
+	c1 = le32dec(authctr0 + 16 + 4*1);
+	c2 = le32dec(authctr0 + 16 + 4*2);
+	c3 = be32dec(authctr0 + 16 + 4*3);
+
+	/*
+	 * In principle we could use REP XCRYPTCTR here, but that
+	 * doesn't help to compute the CBC-MAC step, and certain VIA
+	 * CPUs have some weird errata with REP XCRYPTCTR that make it
+	 * kind of a pain to use.  So let's just use REP XCRYPTECB to
+	 * simultaneously compute the CBC-MAC step and the CTR step.
+	 * (Maybe some VIA CPUs will compute REP XCRYPTECB in parallel,
+	 * who knows...)
+	 */
+	fpu_kern_enter();
+	aesvia_reload_keys();
+	for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+		xor128(authctr, authctr, in);
+		le32enc(authctr + 16 + 4*0, c0);
+		le32enc(authctr + 16 + 4*1, c1);
+		le32enc(authctr + 16 + 4*2, c2);
+		be32enc(authctr + 16 + 4*3, ++c3);
+		aesvia_encN(enc, authctr, authctr, 2, cw0);
+		xor128(out, in, authctr + 16);
+	}
+	fpu_kern_leave();
+
+	if ((uintptr_t)authctr0 & 0xf) {
+		memcpy(authctr0, authctrbuf, 16);
+		explicit_memset(authctrbuf, 0, sizeof authctrbuf);
+	}
+
+	le32enc(authctr0 + 16 + 4*0, c0);
+	le32enc(authctr0 + 16 + 4*1, c1);
+	le32enc(authctr0 + 16 + 4*2, c2);
+	be32enc(authctr0 + 16 + 4*3, c3);
+}
+
+static struct evcnt ccmdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+    NULL, "aesvia", "ccmdec aligned");
+EVCNT_ATTACH_STATIC(ccmdec_aligned_evcnt);
+static struct evcnt ccmdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+    NULL, "aesvia", "ccmdec unaligned");
+EVCNT_ATTACH_STATIC(ccmdec_unaligned_evcnt);
+
+static void
+aesvia_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr0[static 32],
+    uint32_t nrounds)
+{
+	const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+	uint8_t authctrbuf[32] __aligned(16);
+	uint8_t *authctr;
+	uint32_t c0, c1, c2, c3;
+
+	KASSERT(nbytes);
+	KASSERT(nbytes % 16 == 0);
+
+	c0 = le32dec(authctr0 + 16 + 4*0);
+	c1 = le32dec(authctr0 + 16 + 4*1);
+	c2 = le32dec(authctr0 + 16 + 4*2);
+	c3 = be32dec(authctr0 + 16 + 4*3);
+
+	if ((uintptr_t)authctr0 & 0xf) {
+		memcpy(authctrbuf, authctr0, 16);
+		authctr = authctrbuf;
+		le32enc(authctr + 16 + 4*0, c0);
+		le32enc(authctr + 16 + 4*1, c1);
+		le32enc(authctr + 16 + 4*2, c2);
+		ccmdec_unaligned_evcnt.ev_count++;
+	} else {
+		ccmdec_aligned_evcnt.ev_count++;
+	}
+
+	fpu_kern_enter();
+	aesvia_reload_keys();
+	be32enc(authctr + 16 + 4*3, ++c3);
+	aesvia_encN(enc, authctr + 16, authctr + 16, 1, cw0);
+	for (;; in += 16, out += 16) {
+		xor128(out, authctr + 16, in);
+		xor128(authctr, authctr, out);
+		if ((nbytes -= 16) == 0)
+			break;
+		le32enc(authctr + 16 + 4*0, c0);
+		le32enc(authctr + 16 + 4*1, c1);
+		le32enc(authctr + 16 + 4*2, c2);
+		be32enc(authctr + 16 + 4*3, ++c3);
+		aesvia_encN(enc, authctr, authctr, 2, cw0);
+	}
+	aesvia_encN(enc, authctr, authctr, 1, cw0);
+	fpu_kern_leave();
+
+	if ((uintptr_t)authctr0 & 0xf) {
+		memcpy(authctr0, authctrbuf, 16);
+		explicit_memset(authctrbuf, 0, sizeof authctrbuf);
+	}
+
+	le32enc(authctr0 + 16 + 4*0, c0);
+	le32enc(authctr0 + 16 + 4*1, c1);
+	le32enc(authctr0 + 16 + 4*2, c2);
+	be32enc(authctr0 + 16 + 4*3, c3);
+}
+
 static int
 aesvia_probe(void)
 {
@@ -727,4 +897,7 @@ struct aes_impl aes_via_impl = {
 	.ai_cbc_dec = aesvia_cbc_dec,
 	.ai_xts_enc = aesvia_xts_enc,
 	.ai_xts_dec = aesvia_xts_dec,
+	.ai_cbcmac_update1 = aesvia_cbcmac_update1,
+	.ai_ccm_enc1 = aesvia_ccm_enc1,
+	.ai_ccm_dec1 = aesvia_ccm_dec1,
 };

Reply via email to