Module Name: src Committed By: riastradh Date: Mon Jun 29 23:39:31 UTC 2020
Modified Files: src/sys/arch/x86/conf: files.x86 src/sys/arch/x86/x86: identcpu.c Added Files: src/sys/crypto/aes/arch/x86: aes_via.c aes_via.h files.aesvia Log Message: Add AES implementation with VIA ACE. To generate a diff of this commit: cvs rdiff -u -r1.113 -r1.114 src/sys/arch/x86/conf/files.x86 cvs rdiff -u -r1.108 -r1.109 src/sys/arch/x86/x86/identcpu.c cvs rdiff -u -r0 -r1.1 src/sys/crypto/aes/arch/x86/aes_via.c \ src/sys/crypto/aes/arch/x86/aes_via.h \ src/sys/crypto/aes/arch/x86/files.aesvia Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/x86/conf/files.x86 diff -u src/sys/arch/x86/conf/files.x86:1.113 src/sys/arch/x86/conf/files.x86:1.114 --- src/sys/arch/x86/conf/files.x86:1.113 Mon Jun 29 23:38:02 2020 +++ src/sys/arch/x86/conf/files.x86 Mon Jun 29 23:39:30 2020 @@ -1,4 +1,4 @@ -# $NetBSD: files.x86,v 1.113 2020/06/29 23:38:02 riastradh Exp $ +# $NetBSD: files.x86,v 1.114 2020/06/29 23:39:30 riastradh Exp $ # options for MP configuration through the MP spec defflag opt_mpbios.h MPBIOS MPDEBUG MPBIOS_SCANPCI @@ -168,3 +168,6 @@ file arch/x86/pci/pci_addr_fixup.c pci_a # AES-NI include "crypto/aes/arch/x86/files.aesni" + +# VIA ACE +include "crypto/aes/arch/x86/files.aesvia" Index: src/sys/arch/x86/x86/identcpu.c diff -u src/sys/arch/x86/x86/identcpu.c:1.108 src/sys/arch/x86/x86/identcpu.c:1.109 --- src/sys/arch/x86/x86/identcpu.c:1.108 Mon Jun 29 23:29:39 2020 +++ src/sys/arch/x86/x86/identcpu.c Mon Jun 29 23:39:30 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: identcpu.c,v 1.108 2020/06/29 23:29:39 riastradh Exp $ */ +/* $NetBSD: identcpu.c,v 1.109 2020/06/29 23:39:30 riastradh Exp $ */ /*- * Copyright (c) 1999, 2000, 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: identcpu.c,v 1.108 2020/06/29 23:29:39 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: identcpu.c,v 1.109 2020/06/29 23:39:30 riastradh Exp $"); #include "opt_xen.h" @@ -40,6 +40,7 @@ __KERNEL_RCSID(0, "$NetBSD: identcpu.c,v #include <sys/cpu.h> #include <crypto/aes/arch/x86/aes_ni.h> +#include <crypto/aes/arch/x86/aes_via.h> #include <uvm/uvm_extern.h> @@ -1000,7 +1001,10 @@ cpu_probe(struct cpu_info *ci) #ifdef __x86_64__ /* not yet implemented on i386 */ if (cpu_feature[1] & CPUID2_AES) aes_md_init(&aes_ni_impl); + else #endif + if (cpu_feature[4] & CPUID_VIA_HAS_ACE) + aes_md_init(&aes_via_impl); } else { /* * If not first. Warn about cpu_feature mismatch for Added files: Index: src/sys/crypto/aes/arch/x86/aes_via.c diff -u /dev/null src/sys/crypto/aes/arch/x86/aes_via.c:1.1 --- /dev/null Mon Jun 29 23:39:31 2020 +++ src/sys/crypto/aes/arch/x86/aes_via.c Mon Jun 29 23:39:30 2020 @@ -0,0 +1,626 @@ +/* $NetBSD: aes_via.c,v 1.1 2020/06/29 23:39:30 riastradh Exp $ */ + +/*- + * Copyright (c) 2020 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.1 2020/06/29 23:39:30 riastradh Exp $"); + +#include <sys/types.h> +#include <sys/evcnt.h> +#include <sys/systm.h> + +#include <crypto/aes/aes.h> +#include <crypto/aes/aes_bear.h> + +#include <x86/cpufunc.h> +#include <x86/cpuvar.h> +#include <x86/fpu.h> +#include <x86/specialreg.h> +#include <x86/via_padlock.h> + +static void +aesvia_reload_keys(void) +{ + + asm volatile("pushf; popf"); +} + +static uint32_t +aesvia_keylen_cw0(unsigned nrounds) +{ + + /* + * Determine the control word bits for the key size / number of + * rounds. For AES-128, the hardware can do key expansion on + * the fly; for AES-192 and AES-256, software must do it. + */ + switch (nrounds) { + case AES_128_NROUNDS: + return C3_CRYPT_CWLO_KEY128; + case AES_192_NROUNDS: + return C3_CRYPT_CWLO_KEY192 | C3_CRYPT_CWLO_KEYGEN_SW; + case AES_256_NROUNDS: + return C3_CRYPT_CWLO_KEY256 | C3_CRYPT_CWLO_KEYGEN_SW; + default: + panic("invalid AES nrounds: %u", nrounds); + } +} + +static void +aesvia_setenckey(struct aesenc *enc, const uint8_t *key, uint32_t nrounds) +{ + size_t key_len; + + switch (nrounds) { + case AES_128_NROUNDS: + enc->aese_aes.aes_rk[0] = le32dec(key + 4*0); + enc->aese_aes.aes_rk[1] = le32dec(key + 4*1); + enc->aese_aes.aes_rk[2] = le32dec(key + 4*2); + enc->aese_aes.aes_rk[3] = le32dec(key + 4*3); + return; + case AES_192_NROUNDS: + key_len = 24; + break; + case AES_256_NROUNDS: + key_len = 32; + break; + default: + panic("invalid AES nrounds: %u", nrounds); + } + br_aes_ct_keysched_stdenc(enc->aese_aes.aes_rk, key, key_len); +} + +static void +aesvia_setdeckey(struct aesdec *dec, const uint8_t *key, uint32_t nrounds) +{ + size_t key_len; + + switch (nrounds) { + case AES_128_NROUNDS: + dec->aesd_aes.aes_rk[0] = le32dec(key + 4*0); + dec->aesd_aes.aes_rk[1] = le32dec(key + 4*1); + dec->aesd_aes.aes_rk[2] = le32dec(key + 4*2); + dec->aesd_aes.aes_rk[3] = le32dec(key + 4*3); + return; + case AES_192_NROUNDS: + key_len = 24; + break; + case AES_256_NROUNDS: + key_len = 32; + break; + default: + panic("invalid AES nrounds: %u", nrounds); + } + br_aes_ct_keysched_stddec(dec->aesd_aes.aes_rk, key, key_len); +} + +static inline void +aesvia_enc1(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], uint32_t cw0) +{ + const uint32_t cw[4] __aligned(16) = { + [0] = (cw0 + | C3_CRYPT_CWLO_ALG_AES + | C3_CRYPT_CWLO_ENCRYPT + | C3_CRYPT_CWLO_NORMAL), + }; + size_t nblocks = 1; + + KASSERT(((uintptr_t)enc & 0xf) == 0); + KASSERT(((uintptr_t)in & 0xf) == 0); + KASSERT(((uintptr_t)out & 0xf) == 0); + + asm volatile("rep xcryptecb" + : "+c"(nblocks), "+S"(in), "+D"(out) + : "b"(enc), "d"(cw) + : "memory", "cc"); +} + +static inline void +aesvia_dec1(const struct aesdec *dec, const uint8_t in[static 16], + uint8_t out[static 16], uint32_t cw0) +{ + const uint32_t cw[4] __aligned(16) = { + [0] = (cw0 + | C3_CRYPT_CWLO_ALG_AES + | C3_CRYPT_CWLO_DECRYPT + | C3_CRYPT_CWLO_NORMAL), + }; + size_t nblocks = 1; + + KASSERT(((uintptr_t)dec & 0xf) == 0); + KASSERT(((uintptr_t)in & 0xf) == 0); + KASSERT(((uintptr_t)out & 0xf) == 0); + + asm volatile("rep xcryptecb" + : "+c"(nblocks), "+S"(in), "+D"(out) + : "b"(dec), "d"(cw) + : "memory", "cc"); +} + +static struct evcnt enc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "enc aligned"); +EVCNT_ATTACH_STATIC(enc_aligned_evcnt); +static struct evcnt enc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "dec unaligned"); +EVCNT_ATTACH_STATIC(enc_unaligned_evcnt); + +static void +aesvia_enc(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], uint32_t nrounds) +{ + const uint32_t cw0 = aesvia_keylen_cw0(nrounds); + + fpu_kern_enter(); + aesvia_reload_keys(); + if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 && + ((uintptr_t)in & 0xff0) != 0xff0) { + enc_aligned_evcnt.ev_count++; + aesvia_enc1(enc, in, out, cw0); + } else { + enc_unaligned_evcnt.ev_count++; + /* + * VIA requires 16-byte/128-bit alignment, and + * xcrypt-ecb reads one block past the one we're + * working on -- which may go past the end of the page + * into unmapped territory. Use a bounce buffer if + * either constraint is violated. + */ + uint8_t inbuf[16] __aligned(16); + uint8_t outbuf[16] __aligned(16); + + memcpy(inbuf, in, 16); + aesvia_enc1(enc, inbuf, outbuf, cw0); + memcpy(out, outbuf, 16); + + explicit_memset(inbuf, 0, sizeof inbuf); + explicit_memset(outbuf, 0, sizeof outbuf); + } + fpu_kern_leave(); +} + +static struct evcnt dec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "dec aligned"); +EVCNT_ATTACH_STATIC(dec_aligned_evcnt); +static struct evcnt dec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "dec unaligned"); +EVCNT_ATTACH_STATIC(dec_unaligned_evcnt); + +static void +aesvia_dec(const struct aesdec *dec, const uint8_t in[static 16], + uint8_t out[static 16], uint32_t nrounds) +{ + const uint32_t cw0 = aesvia_keylen_cw0(nrounds); + + fpu_kern_enter(); + aesvia_reload_keys(); + if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 && + ((uintptr_t)in & 0xff0) != 0xff0) { + dec_aligned_evcnt.ev_count++; + aesvia_dec1(dec, in, out, cw0); + } else { + dec_unaligned_evcnt.ev_count++; + /* + * VIA requires 16-byte/128-bit alignment, and + * xcrypt-ecb reads one block past the one we're + * working on -- which may go past the end of the page + * into unmapped territory. Use a bounce buffer if + * either constraint is violated. + */ + uint8_t inbuf[16] __aligned(16); + uint8_t outbuf[16] __aligned(16); + + memcpy(inbuf, in, 16); + aesvia_dec1(dec, inbuf, outbuf, cw0); + memcpy(out, outbuf, 16); + + explicit_memset(inbuf, 0, sizeof inbuf); + explicit_memset(outbuf, 0, sizeof outbuf); + } + fpu_kern_leave(); +} + +static inline void +aesvia_cbc_enc1(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nblocks, uint8_t **ivp, uint32_t cw0) +{ + const uint32_t cw[4] __aligned(16) = { + [0] = (cw0 + | C3_CRYPT_CWLO_ALG_AES + | C3_CRYPT_CWLO_ENCRYPT + | C3_CRYPT_CWLO_NORMAL), + }; + + KASSERT(((uintptr_t)enc & 0xf) == 0); + KASSERT(((uintptr_t)in & 0xf) == 0); + KASSERT(((uintptr_t)out & 0xf) == 0); + KASSERT(((uintptr_t)*ivp & 0xf) == 0); + + /* + * Register effects: + * - Counts nblocks down to zero. + * - Advances in by nblocks (units of blocks). + * - Advances out by nblocks (units of blocks). + * - Updates *ivp to point at the last block of out. + */ + asm volatile("rep xcryptcbc" + : "+c"(nblocks), "+S"(in), "+D"(out), "+a"(*ivp) + : "b"(enc), "d"(cw) + : "memory", "cc"); +} + +static inline void +aesvia_cbc_dec1(const struct aesdec *dec, const uint8_t in[static 16], + uint8_t out[static 16], size_t nblocks, uint8_t iv[static 16], + uint32_t cw0) +{ + const uint32_t cw[4] __aligned(16) = { + [0] = (cw0 + | C3_CRYPT_CWLO_ALG_AES + | C3_CRYPT_CWLO_DECRYPT + | C3_CRYPT_CWLO_NORMAL), + }; + + KASSERT(((uintptr_t)dec & 0xf) == 0); + KASSERT(((uintptr_t)in & 0xf) == 0); + KASSERT(((uintptr_t)out & 0xf) == 0); + KASSERT(((uintptr_t)iv & 0xf) == 0); + + /* + * Register effects: + * - Counts nblocks down to zero. + * - Advances in by nblocks (units of blocks). + * - Advances out by nblocks (units of blocks). + * Memory side effects: + * - Writes what was the last block of in at the address iv. + */ + asm volatile("rep xcryptcbc" + : "+c"(nblocks), "+S"(in), "+D"(out) + : "a"(iv), "b"(dec), "d"(cw) + : "memory", "cc"); +} + +static inline void +xor128(void *x, const void *a, const void *b) +{ + uint32_t *x32 = x; + const uint32_t *a32 = a; + const uint32_t *b32 = b; + + x32[0] = a32[0] ^ b32[0]; + x32[1] = a32[1] ^ b32[1]; + x32[2] = a32[2] ^ b32[2]; + x32[3] = a32[3] ^ b32[3]; +} + +static struct evcnt cbcenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "cbcenc aligned"); +EVCNT_ATTACH_STATIC(cbcenc_aligned_evcnt); +static struct evcnt cbcenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "cbcenc unaligned"); +EVCNT_ATTACH_STATIC(cbcenc_unaligned_evcnt); + +static void +aesvia_cbc_enc(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], + uint32_t nrounds) +{ + const uint32_t cw0 = aesvia_keylen_cw0(nrounds); + + KASSERT(nbytes % 16 == 0); + if (nbytes == 0) + return; + + fpu_kern_enter(); + aesvia_reload_keys(); + if ((((uintptr_t)in | (uintptr_t)out | (uintptr_t)iv) & 0xf) == 0) { + cbcenc_aligned_evcnt.ev_count++; + uint8_t *ivp = iv; + aesvia_cbc_enc1(enc, in, out, nbytes/16, &ivp, cw0); + memcpy(iv, ivp, 16); + } else { + cbcenc_unaligned_evcnt.ev_count++; + uint8_t cv[16] __aligned(16); + uint8_t tmp[16] __aligned(16); + + memcpy(cv, iv, 16); + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + memcpy(tmp, in, 16); + xor128(tmp, tmp, cv); + aesvia_enc1(enc, tmp, cv, cw0); + memcpy(out, cv, 16); + } + memcpy(iv, cv, 16); + } + fpu_kern_leave(); +} + +static struct evcnt cbcdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "cbcdec aligned"); +EVCNT_ATTACH_STATIC(cbcdec_aligned_evcnt); +static struct evcnt cbcdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "cbcdec unaligned"); +EVCNT_ATTACH_STATIC(cbcdec_unaligned_evcnt); + +static void +aesvia_cbc_dec(const struct aesdec *dec, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], + uint32_t nrounds) +{ + const uint32_t cw0 = aesvia_keylen_cw0(nrounds); + + KASSERT(nbytes % 16 == 0); + if (nbytes == 0) + return; + + fpu_kern_enter(); + aesvia_reload_keys(); + if ((((uintptr_t)in | (uintptr_t)out | (uintptr_t)iv) & 0xf) == 0) { + cbcdec_aligned_evcnt.ev_count++; + aesvia_cbc_dec1(dec, in, out, nbytes/16, iv, cw0); + } else { + cbcdec_unaligned_evcnt.ev_count++; + uint8_t iv0[16] __aligned(16); + uint8_t cv[16] __aligned(16); + uint8_t tmp[16] __aligned(16); + + memcpy(iv0, iv, 16); + memcpy(cv, in + nbytes - 16, 16); + memcpy(iv, cv, 16); + + for (;;) { + aesvia_dec1(dec, cv, tmp, cw0); + if ((nbytes -= 16) == 0) + break; + memcpy(cv, in + nbytes - 16, 16); + xor128(tmp, tmp, cv); + memcpy(out + nbytes, tmp, 16); + } + + xor128(tmp, tmp, iv0); + memcpy(out, tmp, 16); + explicit_memset(tmp, 0, sizeof tmp); + } + fpu_kern_leave(); +} + +static inline void +aesvia_xts_update(uint32_t *t0, uint32_t *t1, uint32_t *t2, uint32_t *t3) +{ + uint32_t s0, s1, s2, s3; + + s0 = *t0 >> 31; + s1 = *t1 >> 31; + s2 = *t2 >> 31; + s3 = *t3 >> 31; + *t0 = (*t0 << 1) ^ (-s3 & 0x87); + *t1 = (*t1 << 1) ^ s0; + *t2 = (*t2 << 1) ^ s1; + *t3 = (*t3 << 1) ^ s2; +} + +static int +aesvia_xts_update_selftest(void) +{ + static const struct { + uint32_t in[4], out[4]; + } cases[] = { + { {1}, {2} }, + { {0x80000000U,0,0,0}, {0,1,0,0} }, + { {0,0x80000000U,0,0}, {0,0,1,0} }, + { {0,0,0x80000000U,0}, {0,0,0,1} }, + { {0,0,0,0x80000000U}, {0x87,0,0,0} }, + { {0,0x80000000U,0,0x80000000U}, {0x87,0,1,0} }, + }; + unsigned i; + uint32_t t0, t1, t2, t3; + + for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) { + t0 = cases[i].in[0]; + t1 = cases[i].in[1]; + t2 = cases[i].in[2]; + t3 = cases[i].in[3]; + aesvia_xts_update(&t0, &t1, &t2, &t3); + if (t0 != cases[i].out[0] || + t1 != cases[i].out[1] || + t2 != cases[i].out[2] || + t3 != cases[i].out[3]) + return -1; + } + + /* Success! */ + return 0; +} + +static struct evcnt xtsenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "xtsenc aligned"); +EVCNT_ATTACH_STATIC(xtsenc_aligned_evcnt); +static struct evcnt xtsenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "xtsenc unaligned"); +EVCNT_ATTACH_STATIC(xtsenc_unaligned_evcnt); + +static void +aesvia_xts_enc(const struct aesenc *enc, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16], + uint32_t nrounds) +{ + const uint32_t cw0 = aesvia_keylen_cw0(nrounds); + uint32_t t[4]; + + KASSERT(nbytes % 16 == 0); + + memcpy(t, tweak, 16); + + fpu_kern_enter(); + aesvia_reload_keys(); + if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0) { + xtsenc_aligned_evcnt.ev_count++; + unsigned lastblock = 0; + + /* + * Make sure the last block is not the last block of a + * page. (Note that we store the AES input in `out' as + * a temporary buffer, rather than reading it directly + * from `in', since we have to combine the tweak + * first.) + */ + lastblock = 16*(((uintptr_t)(out + nbytes) & 0xfff) == 0); + nbytes -= lastblock; + + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + xor128(out, in, t); + aesvia_enc1(enc, out, out, cw0); + xor128(out, out, t); + aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); + } + + /* Handle the last block of a page, if necessary. */ + if (lastblock) { + uint8_t buf[16] __aligned(16); + xor128(buf, in, t); + aesvia_enc1(enc, buf, out, cw0); + explicit_memset(buf, 0, sizeof buf); + } + } else { + xtsenc_unaligned_evcnt.ev_count++; + uint8_t buf[16] __aligned(16); + + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + memcpy(buf, in, 16); + xor128(buf, buf, t); + aesvia_enc1(enc, buf, buf, cw0); + xor128(buf, buf, t); + memcpy(out, buf, 16); + aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); + } + + explicit_memset(buf, 0, sizeof buf); + } + fpu_kern_leave(); + + memcpy(tweak, t, 16); + explicit_memset(t, 0, sizeof t); +} + +static struct evcnt xtsdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "xtsdec aligned"); +EVCNT_ATTACH_STATIC(xtsdec_aligned_evcnt); +static struct evcnt xtsdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, + NULL, "aesvia", "xtsdec unaligned"); +EVCNT_ATTACH_STATIC(xtsdec_unaligned_evcnt); + +static void +aesvia_xts_dec(const struct aesdec *dec, const uint8_t in[static 16], + uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16], + uint32_t nrounds) +{ + const uint32_t cw0 = aesvia_keylen_cw0(nrounds); + uint32_t t[4]; + + KASSERT(nbytes % 16 == 0); + + memcpy(t, tweak, 16); + + fpu_kern_enter(); + aesvia_reload_keys(); + if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0) { + xtsdec_aligned_evcnt.ev_count++; + unsigned lastblock = 0; + + /* + * Make sure the last block is not the last block of a + * page. (Note that we store the AES input in `out' as + * a temporary buffer, rather than reading it directly + * from `in', since we have to combine the tweak + * first.) + */ + lastblock = 16*(((uintptr_t)(out + nbytes) & 0xfff) == 0); + nbytes -= lastblock; + + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + xor128(out, in, t); + aesvia_dec1(dec, out, out, cw0); + xor128(out, out, t); + aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); + } + + /* Handle the last block of a page, if necessary. */ + if (lastblock) { + uint8_t buf[16] __aligned(16); + xor128(buf, in, t); + aesvia_dec1(dec, buf, out, cw0); + explicit_memset(buf, 0, sizeof buf); + } + } else { + xtsdec_unaligned_evcnt.ev_count++; + uint8_t buf[16] __aligned(16); + + for (; nbytes; nbytes -= 16, in += 16, out += 16) { + memcpy(buf, in, 16); + xor128(buf, buf, t); + aesvia_dec1(dec, buf, buf, cw0); + xor128(buf, buf, t); + memcpy(out, buf, 16); + aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); + } + + explicit_memset(buf, 0, sizeof buf); + } + fpu_kern_leave(); + + memcpy(tweak, t, 16); + explicit_memset(t, 0, sizeof t); +} + +static int +aesvia_probe(void) +{ + + /* Verify that the CPU advertises VIA ACE support. */ + if ((cpu_feature[4] & CPUID_VIA_HAS_ACE) == 0) + return -1; + + /* Verify that our XTS tweak update logic works. */ + if (aesvia_xts_update_selftest()) + return -1; + + /* Success! */ + return 0; +} + +struct aes_impl aes_via_impl = { + .ai_name = "VIA ACE", + .ai_probe = aesvia_probe, + .ai_setenckey = aesvia_setenckey, + .ai_setdeckey = aesvia_setdeckey, + .ai_enc = aesvia_enc, + .ai_dec = aesvia_dec, + .ai_cbc_enc = aesvia_cbc_enc, + .ai_cbc_dec = aesvia_cbc_dec, + .ai_xts_enc = aesvia_xts_enc, + .ai_xts_dec = aesvia_xts_dec, +}; Index: src/sys/crypto/aes/arch/x86/aes_via.h diff -u /dev/null src/sys/crypto/aes/arch/x86/aes_via.h:1.1 --- /dev/null Mon Jun 29 23:39:31 2020 +++ src/sys/crypto/aes/arch/x86/aes_via.h Mon Jun 29 23:39:30 2020 @@ -0,0 +1,36 @@ +/* $NetBSD: aes_via.h,v 1.1 2020/06/29 23:39:30 riastradh Exp $ */ + +/*- + * Copyright (c) 2020 The NetBSD Foundation, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CRYPTO_AES_ARCH_X86_AES_VIA_H +#define _CRYPTO_AES_ARCH_X86_AES_VIA_H + +#include <crypto/aes/aes.h> + +extern struct aes_impl aes_via_impl; + +#endif /* _CRYPTO_AES_ARCH_X86_AES_VIA_H */ Index: src/sys/crypto/aes/arch/x86/files.aesvia diff -u /dev/null src/sys/crypto/aes/arch/x86/files.aesvia:1.1 --- /dev/null Mon Jun 29 23:39:31 2020 +++ src/sys/crypto/aes/arch/x86/files.aesvia Mon Jun 29 23:39:30 2020 @@ -0,0 +1,3 @@ +# $NetBSD: files.aesvia,v 1.1 2020/06/29 23:39:30 riastradh Exp $ + +file crypto/aes/arch/x86/aes_via.c aes