Module Name: src
Committed By: riastradh
Date: Mon Jun 29 23:39:31 UTC 2020
Modified Files:
src/sys/arch/x86/conf: files.x86
src/sys/arch/x86/x86: identcpu.c
Added Files:
src/sys/crypto/aes/arch/x86: aes_via.c aes_via.h files.aesvia
Log Message:
Add AES implementation with VIA ACE.
To generate a diff of this commit:
cvs rdiff -u -r1.113 -r1.114 src/sys/arch/x86/conf/files.x86
cvs rdiff -u -r1.108 -r1.109 src/sys/arch/x86/x86/identcpu.c
cvs rdiff -u -r0 -r1.1 src/sys/crypto/aes/arch/x86/aes_via.c \
src/sys/crypto/aes/arch/x86/aes_via.h \
src/sys/crypto/aes/arch/x86/files.aesvia
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/conf/files.x86
diff -u src/sys/arch/x86/conf/files.x86:1.113 src/sys/arch/x86/conf/files.x86:1.114
--- src/sys/arch/x86/conf/files.x86:1.113 Mon Jun 29 23:38:02 2020
+++ src/sys/arch/x86/conf/files.x86 Mon Jun 29 23:39:30 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.x86,v 1.113 2020/06/29 23:38:02 riastradh Exp $
+# $NetBSD: files.x86,v 1.114 2020/06/29 23:39:30 riastradh Exp $
# options for MP configuration through the MP spec
defflag opt_mpbios.h MPBIOS MPDEBUG MPBIOS_SCANPCI
@@ -168,3 +168,6 @@ file arch/x86/pci/pci_addr_fixup.c pci_a
# AES-NI
include "crypto/aes/arch/x86/files.aesni"
+
+# VIA ACE
+include "crypto/aes/arch/x86/files.aesvia"
Index: src/sys/arch/x86/x86/identcpu.c
diff -u src/sys/arch/x86/x86/identcpu.c:1.108 src/sys/arch/x86/x86/identcpu.c:1.109
--- src/sys/arch/x86/x86/identcpu.c:1.108 Mon Jun 29 23:29:39 2020
+++ src/sys/arch/x86/x86/identcpu.c Mon Jun 29 23:39:30 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: identcpu.c,v 1.108 2020/06/29 23:29:39 riastradh Exp $ */
+/* $NetBSD: identcpu.c,v 1.109 2020/06/29 23:39:30 riastradh Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: identcpu.c,v 1.108 2020/06/29 23:29:39 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: identcpu.c,v 1.109 2020/06/29 23:39:30 riastradh Exp $");
#include "opt_xen.h"
@@ -40,6 +40,7 @@ __KERNEL_RCSID(0, "$NetBSD: identcpu.c,v
#include <sys/cpu.h>
#include <crypto/aes/arch/x86/aes_ni.h>
+#include <crypto/aes/arch/x86/aes_via.h>
#include <uvm/uvm_extern.h>
@@ -1000,7 +1001,10 @@ cpu_probe(struct cpu_info *ci)
#ifdef __x86_64__ /* not yet implemented on i386 */
if (cpu_feature[1] & CPUID2_AES)
aes_md_init(&aes_ni_impl);
+ else
#endif
+ if (cpu_feature[4] & CPUID_VIA_HAS_ACE)
+ aes_md_init(&aes_via_impl);
} else {
/*
* If not first. Warn about cpu_feature mismatch for
Added files:
Index: src/sys/crypto/aes/arch/x86/aes_via.c
diff -u /dev/null src/sys/crypto/aes/arch/x86/aes_via.c:1.1
--- /dev/null Mon Jun 29 23:39:31 2020
+++ src/sys/crypto/aes/arch/x86/aes_via.c Mon Jun 29 23:39:30 2020
@@ -0,0 +1,626 @@
+/* $NetBSD: aes_via.c,v 1.1 2020/06/29 23:39:30 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.1 2020/06/29 23:39:30 riastradh Exp $");
+
+#include <sys/types.h>
+#include <sys/evcnt.h>
+#include <sys/systm.h>
+
+#include <crypto/aes/aes.h>
+#include <crypto/aes/aes_bear.h>
+
+#include <x86/cpufunc.h>
+#include <x86/cpuvar.h>
+#include <x86/fpu.h>
+#include <x86/specialreg.h>
+#include <x86/via_padlock.h>
+
+static void
+aesvia_reload_keys(void)
+{
+
+ asm volatile("pushf; popf");
+}
+
+static uint32_t
+aesvia_keylen_cw0(unsigned nrounds)
+{
+
+ /*
+ * Determine the control word bits for the key size / number of
+ * rounds. For AES-128, the hardware can do key expansion on
+ * the fly; for AES-192 and AES-256, software must do it.
+ */
+ switch (nrounds) {
+ case AES_128_NROUNDS:
+ return C3_CRYPT_CWLO_KEY128;
+ case AES_192_NROUNDS:
+ return C3_CRYPT_CWLO_KEY192 | C3_CRYPT_CWLO_KEYGEN_SW;
+ case AES_256_NROUNDS:
+ return C3_CRYPT_CWLO_KEY256 | C3_CRYPT_CWLO_KEYGEN_SW;
+ default:
+ panic("invalid AES nrounds: %u", nrounds);
+ }
+}
+
+static void
+aesvia_setenckey(struct aesenc *enc, const uint8_t *key, uint32_t nrounds)
+{
+ size_t key_len;
+
+ switch (nrounds) {
+ case AES_128_NROUNDS:
+ enc->aese_aes.aes_rk[0] = le32dec(key + 4*0);
+ enc->aese_aes.aes_rk[1] = le32dec(key + 4*1);
+ enc->aese_aes.aes_rk[2] = le32dec(key + 4*2);
+ enc->aese_aes.aes_rk[3] = le32dec(key + 4*3);
+ return;
+ case AES_192_NROUNDS:
+ key_len = 24;
+ break;
+ case AES_256_NROUNDS:
+ key_len = 32;
+ break;
+ default:
+ panic("invalid AES nrounds: %u", nrounds);
+ }
+ br_aes_ct_keysched_stdenc(enc->aese_aes.aes_rk, key, key_len);
+}
+
+static void
+aesvia_setdeckey(struct aesdec *dec, const uint8_t *key, uint32_t nrounds)
+{
+ size_t key_len;
+
+ switch (nrounds) {
+ case AES_128_NROUNDS:
+ dec->aesd_aes.aes_rk[0] = le32dec(key + 4*0);
+ dec->aesd_aes.aes_rk[1] = le32dec(key + 4*1);
+ dec->aesd_aes.aes_rk[2] = le32dec(key + 4*2);
+ dec->aesd_aes.aes_rk[3] = le32dec(key + 4*3);
+ return;
+ case AES_192_NROUNDS:
+ key_len = 24;
+ break;
+ case AES_256_NROUNDS:
+ key_len = 32;
+ break;
+ default:
+ panic("invalid AES nrounds: %u", nrounds);
+ }
+ br_aes_ct_keysched_stddec(dec->aesd_aes.aes_rk, key, key_len);
+}
+
+static inline void
+aesvia_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_ENCRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+ size_t nblocks = 1;
+
+ KASSERT(((uintptr_t)enc & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+
+ asm volatile("rep xcryptecb"
+ : "+c"(nblocks), "+S"(in), "+D"(out)
+ : "b"(enc), "d"(cw)
+ : "memory", "cc");
+}
+
+static inline void
+aesvia_dec1(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_DECRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+ size_t nblocks = 1;
+
+ KASSERT(((uintptr_t)dec & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+
+ asm volatile("rep xcryptecb"
+ : "+c"(nblocks), "+S"(in), "+D"(out)
+ : "b"(dec), "d"(cw)
+ : "memory", "cc");
+}
+
+static struct evcnt enc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "enc aligned");
+EVCNT_ATTACH_STATIC(enc_aligned_evcnt);
+static struct evcnt enc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "dec unaligned");
+EVCNT_ATTACH_STATIC(enc_unaligned_evcnt);
+
+static void
+aesvia_enc(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 &&
+ ((uintptr_t)in & 0xff0) != 0xff0) {
+ enc_aligned_evcnt.ev_count++;
+ aesvia_enc1(enc, in, out, cw0);
+ } else {
+ enc_unaligned_evcnt.ev_count++;
+ /*
+ * VIA requires 16-byte/128-bit alignment, and
+ * xcrypt-ecb reads one block past the one we're
+ * working on -- which may go past the end of the page
+ * into unmapped territory. Use a bounce buffer if
+ * either constraint is violated.
+ */
+ uint8_t inbuf[16] __aligned(16);
+ uint8_t outbuf[16] __aligned(16);
+
+ memcpy(inbuf, in, 16);
+ aesvia_enc1(enc, inbuf, outbuf, cw0);
+ memcpy(out, outbuf, 16);
+
+ explicit_memset(inbuf, 0, sizeof inbuf);
+ explicit_memset(outbuf, 0, sizeof outbuf);
+ }
+ fpu_kern_leave();
+}
+
+static struct evcnt dec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "dec aligned");
+EVCNT_ATTACH_STATIC(dec_aligned_evcnt);
+static struct evcnt dec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "dec unaligned");
+EVCNT_ATTACH_STATIC(dec_unaligned_evcnt);
+
+static void
+aesvia_dec(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 &&
+ ((uintptr_t)in & 0xff0) != 0xff0) {
+ dec_aligned_evcnt.ev_count++;
+ aesvia_dec1(dec, in, out, cw0);
+ } else {
+ dec_unaligned_evcnt.ev_count++;
+ /*
+ * VIA requires 16-byte/128-bit alignment, and
+ * xcrypt-ecb reads one block past the one we're
+ * working on -- which may go past the end of the page
+ * into unmapped territory. Use a bounce buffer if
+ * either constraint is violated.
+ */
+ uint8_t inbuf[16] __aligned(16);
+ uint8_t outbuf[16] __aligned(16);
+
+ memcpy(inbuf, in, 16);
+ aesvia_dec1(dec, inbuf, outbuf, cw0);
+ memcpy(out, outbuf, 16);
+
+ explicit_memset(inbuf, 0, sizeof inbuf);
+ explicit_memset(outbuf, 0, sizeof outbuf);
+ }
+ fpu_kern_leave();
+}
+
+static inline void
+aesvia_cbc_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nblocks, uint8_t **ivp, uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_ENCRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+
+ KASSERT(((uintptr_t)enc & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+ KASSERT(((uintptr_t)*ivp & 0xf) == 0);
+
+ /*
+ * Register effects:
+ * - Counts nblocks down to zero.
+ * - Advances in by nblocks (units of blocks).
+ * - Advances out by nblocks (units of blocks).
+ * - Updates *ivp to point at the last block of out.
+ */
+ asm volatile("rep xcryptcbc"
+ : "+c"(nblocks), "+S"(in), "+D"(out), "+a"(*ivp)
+ : "b"(enc), "d"(cw)
+ : "memory", "cc");
+}
+
+static inline void
+aesvia_cbc_dec1(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nblocks, uint8_t iv[static 16],
+ uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_DECRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+
+ KASSERT(((uintptr_t)dec & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+ KASSERT(((uintptr_t)iv & 0xf) == 0);
+
+ /*
+ * Register effects:
+ * - Counts nblocks down to zero.
+ * - Advances in by nblocks (units of blocks).
+ * - Advances out by nblocks (units of blocks).
+ * Memory side effects:
+ * - Writes what was the last block of in at the address iv.
+ */
+ asm volatile("rep xcryptcbc"
+ : "+c"(nblocks), "+S"(in), "+D"(out)
+ : "a"(iv), "b"(dec), "d"(cw)
+ : "memory", "cc");
+}
+
+static inline void
+xor128(void *x, const void *a, const void *b)
+{
+ uint32_t *x32 = x;
+ const uint32_t *a32 = a;
+ const uint32_t *b32 = b;
+
+ x32[0] = a32[0] ^ b32[0];
+ x32[1] = a32[1] ^ b32[1];
+ x32[2] = a32[2] ^ b32[2];
+ x32[3] = a32[3] ^ b32[3];
+}
+
+static struct evcnt cbcenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "cbcenc aligned");
+EVCNT_ATTACH_STATIC(cbcenc_aligned_evcnt);
+static struct evcnt cbcenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "cbcenc unaligned");
+EVCNT_ATTACH_STATIC(cbcenc_unaligned_evcnt);
+
+static void
+aesvia_cbc_enc(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
+ uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+
+ KASSERT(nbytes % 16 == 0);
+ if (nbytes == 0)
+ return;
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out | (uintptr_t)iv) & 0xf) == 0) {
+ cbcenc_aligned_evcnt.ev_count++;
+ uint8_t *ivp = iv;
+ aesvia_cbc_enc1(enc, in, out, nbytes/16, &ivp, cw0);
+ memcpy(iv, ivp, 16);
+ } else {
+ cbcenc_unaligned_evcnt.ev_count++;
+ uint8_t cv[16] __aligned(16);
+ uint8_t tmp[16] __aligned(16);
+
+ memcpy(cv, iv, 16);
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ memcpy(tmp, in, 16);
+ xor128(tmp, tmp, cv);
+ aesvia_enc1(enc, tmp, cv, cw0);
+ memcpy(out, cv, 16);
+ }
+ memcpy(iv, cv, 16);
+ }
+ fpu_kern_leave();
+}
+
+static struct evcnt cbcdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "cbcdec aligned");
+EVCNT_ATTACH_STATIC(cbcdec_aligned_evcnt);
+static struct evcnt cbcdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "cbcdec unaligned");
+EVCNT_ATTACH_STATIC(cbcdec_unaligned_evcnt);
+
+static void
+aesvia_cbc_dec(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
+ uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+
+ KASSERT(nbytes % 16 == 0);
+ if (nbytes == 0)
+ return;
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out | (uintptr_t)iv) & 0xf) == 0) {
+ cbcdec_aligned_evcnt.ev_count++;
+ aesvia_cbc_dec1(dec, in, out, nbytes/16, iv, cw0);
+ } else {
+ cbcdec_unaligned_evcnt.ev_count++;
+ uint8_t iv0[16] __aligned(16);
+ uint8_t cv[16] __aligned(16);
+ uint8_t tmp[16] __aligned(16);
+
+ memcpy(iv0, iv, 16);
+ memcpy(cv, in + nbytes - 16, 16);
+ memcpy(iv, cv, 16);
+
+ for (;;) {
+ aesvia_dec1(dec, cv, tmp, cw0);
+ if ((nbytes -= 16) == 0)
+ break;
+ memcpy(cv, in + nbytes - 16, 16);
+ xor128(tmp, tmp, cv);
+ memcpy(out + nbytes, tmp, 16);
+ }
+
+ xor128(tmp, tmp, iv0);
+ memcpy(out, tmp, 16);
+ explicit_memset(tmp, 0, sizeof tmp);
+ }
+ fpu_kern_leave();
+}
+
+static inline void
+aesvia_xts_update(uint32_t *t0, uint32_t *t1, uint32_t *t2, uint32_t *t3)
+{
+ uint32_t s0, s1, s2, s3;
+
+ s0 = *t0 >> 31;
+ s1 = *t1 >> 31;
+ s2 = *t2 >> 31;
+ s3 = *t3 >> 31;
+ *t0 = (*t0 << 1) ^ (-s3 & 0x87);
+ *t1 = (*t1 << 1) ^ s0;
+ *t2 = (*t2 << 1) ^ s1;
+ *t3 = (*t3 << 1) ^ s2;
+}
+
+static int
+aesvia_xts_update_selftest(void)
+{
+ static const struct {
+ uint32_t in[4], out[4];
+ } cases[] = {
+ { {1}, {2} },
+ { {0x80000000U,0,0,0}, {0,1,0,0} },
+ { {0,0x80000000U,0,0}, {0,0,1,0} },
+ { {0,0,0x80000000U,0}, {0,0,0,1} },
+ { {0,0,0,0x80000000U}, {0x87,0,0,0} },
+ { {0,0x80000000U,0,0x80000000U}, {0x87,0,1,0} },
+ };
+ unsigned i;
+ uint32_t t0, t1, t2, t3;
+
+ for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) {
+ t0 = cases[i].in[0];
+ t1 = cases[i].in[1];
+ t2 = cases[i].in[2];
+ t3 = cases[i].in[3];
+ aesvia_xts_update(&t0, &t1, &t2, &t3);
+ if (t0 != cases[i].out[0] ||
+ t1 != cases[i].out[1] ||
+ t2 != cases[i].out[2] ||
+ t3 != cases[i].out[3])
+ return -1;
+ }
+
+ /* Success! */
+ return 0;
+}
+
+static struct evcnt xtsenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "xtsenc aligned");
+EVCNT_ATTACH_STATIC(xtsenc_aligned_evcnt);
+static struct evcnt xtsenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "xtsenc unaligned");
+EVCNT_ATTACH_STATIC(xtsenc_unaligned_evcnt);
+
+static void
+aesvia_xts_enc(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16],
+ uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+ uint32_t t[4];
+
+ KASSERT(nbytes % 16 == 0);
+
+ memcpy(t, tweak, 16);
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0) {
+ xtsenc_aligned_evcnt.ev_count++;
+ unsigned lastblock = 0;
+
+ /*
+ * Make sure the last block is not the last block of a
+ * page. (Note that we store the AES input in `out' as
+ * a temporary buffer, rather than reading it directly
+ * from `in', since we have to combine the tweak
+ * first.)
+ */
+ lastblock = 16*(((uintptr_t)(out + nbytes) & 0xfff) == 0);
+ nbytes -= lastblock;
+
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ xor128(out, in, t);
+ aesvia_enc1(enc, out, out, cw0);
+ xor128(out, out, t);
+ aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]);
+ }
+
+ /* Handle the last block of a page, if necessary. */
+ if (lastblock) {
+ uint8_t buf[16] __aligned(16);
+ xor128(buf, in, t);
+ aesvia_enc1(enc, buf, out, cw0);
+ explicit_memset(buf, 0, sizeof buf);
+ }
+ } else {
+ xtsenc_unaligned_evcnt.ev_count++;
+ uint8_t buf[16] __aligned(16);
+
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ memcpy(buf, in, 16);
+ xor128(buf, buf, t);
+ aesvia_enc1(enc, buf, buf, cw0);
+ xor128(buf, buf, t);
+ memcpy(out, buf, 16);
+ aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]);
+ }
+
+ explicit_memset(buf, 0, sizeof buf);
+ }
+ fpu_kern_leave();
+
+ memcpy(tweak, t, 16);
+ explicit_memset(t, 0, sizeof t);
+}
+
+static struct evcnt xtsdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "xtsdec aligned");
+EVCNT_ATTACH_STATIC(xtsdec_aligned_evcnt);
+static struct evcnt xtsdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "xtsdec unaligned");
+EVCNT_ATTACH_STATIC(xtsdec_unaligned_evcnt);
+
+static void
+aesvia_xts_dec(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16],
+ uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+ uint32_t t[4];
+
+ KASSERT(nbytes % 16 == 0);
+
+ memcpy(t, tweak, 16);
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0) {
+ xtsdec_aligned_evcnt.ev_count++;
+ unsigned lastblock = 0;
+
+ /*
+ * Make sure the last block is not the last block of a
+ * page. (Note that we store the AES input in `out' as
+ * a temporary buffer, rather than reading it directly
+ * from `in', since we have to combine the tweak
+ * first.)
+ */
+ lastblock = 16*(((uintptr_t)(out + nbytes) & 0xfff) == 0);
+ nbytes -= lastblock;
+
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ xor128(out, in, t);
+ aesvia_dec1(dec, out, out, cw0);
+ xor128(out, out, t);
+ aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]);
+ }
+
+ /* Handle the last block of a page, if necessary. */
+ if (lastblock) {
+ uint8_t buf[16] __aligned(16);
+ xor128(buf, in, t);
+ aesvia_dec1(dec, buf, out, cw0);
+ explicit_memset(buf, 0, sizeof buf);
+ }
+ } else {
+ xtsdec_unaligned_evcnt.ev_count++;
+ uint8_t buf[16] __aligned(16);
+
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ memcpy(buf, in, 16);
+ xor128(buf, buf, t);
+ aesvia_dec1(dec, buf, buf, cw0);
+ xor128(buf, buf, t);
+ memcpy(out, buf, 16);
+ aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]);
+ }
+
+ explicit_memset(buf, 0, sizeof buf);
+ }
+ fpu_kern_leave();
+
+ memcpy(tweak, t, 16);
+ explicit_memset(t, 0, sizeof t);
+}
+
+static int
+aesvia_probe(void)
+{
+
+ /* Verify that the CPU advertises VIA ACE support. */
+ if ((cpu_feature[4] & CPUID_VIA_HAS_ACE) == 0)
+ return -1;
+
+ /* Verify that our XTS tweak update logic works. */
+ if (aesvia_xts_update_selftest())
+ return -1;
+
+ /* Success! */
+ return 0;
+}
+
+struct aes_impl aes_via_impl = {
+ .ai_name = "VIA ACE",
+ .ai_probe = aesvia_probe,
+ .ai_setenckey = aesvia_setenckey,
+ .ai_setdeckey = aesvia_setdeckey,
+ .ai_enc = aesvia_enc,
+ .ai_dec = aesvia_dec,
+ .ai_cbc_enc = aesvia_cbc_enc,
+ .ai_cbc_dec = aesvia_cbc_dec,
+ .ai_xts_enc = aesvia_xts_enc,
+ .ai_xts_dec = aesvia_xts_dec,
+};
Index: src/sys/crypto/aes/arch/x86/aes_via.h
diff -u /dev/null src/sys/crypto/aes/arch/x86/aes_via.h:1.1
--- /dev/null Mon Jun 29 23:39:31 2020
+++ src/sys/crypto/aes/arch/x86/aes_via.h Mon Jun 29 23:39:30 2020
@@ -0,0 +1,36 @@
+/* $NetBSD: aes_via.h,v 1.1 2020/06/29 23:39:30 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _CRYPTO_AES_ARCH_X86_AES_VIA_H
+#define _CRYPTO_AES_ARCH_X86_AES_VIA_H
+
+#include <crypto/aes/aes.h>
+
+extern struct aes_impl aes_via_impl;
+
+#endif /* _CRYPTO_AES_ARCH_X86_AES_VIA_H */
Index: src/sys/crypto/aes/arch/x86/files.aesvia
diff -u /dev/null src/sys/crypto/aes/arch/x86/files.aesvia:1.1
--- /dev/null Mon Jun 29 23:39:31 2020
+++ src/sys/crypto/aes/arch/x86/files.aesvia Mon Jun 29 23:39:30 2020
@@ -0,0 +1,3 @@
+# $NetBSD: files.aesvia,v 1.1 2020/06/29 23:39:30 riastradh Exp $
+
+file crypto/aes/arch/x86/aes_via.c aes
