Module Name: src
Committed By: riastradh
Date: Sat Jul 25 22:29:06 UTC 2020
Modified Files:
src/sys/crypto/aes/arch/x86: aes_ni.c aes_ni.h aes_ni_64.S
Log Message:
Implement AES-CCM with x86 AES-NI.
To generate a diff of this commit:
cvs rdiff -u -r1.3 -r1.4 src/sys/crypto/aes/arch/x86/aes_ni.c \
src/sys/crypto/aes/arch/x86/aes_ni_64.S
cvs rdiff -u -r1.2 -r1.3 src/sys/crypto/aes/arch/x86/aes_ni.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/crypto/aes/arch/x86/aes_ni.c
diff -u src/sys/crypto/aes/arch/x86/aes_ni.c:1.3 src/sys/crypto/aes/arch/x86/aes_ni.c:1.4
--- src/sys/crypto/aes/arch/x86/aes_ni.c:1.3 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_ni.c Sat Jul 25 22:29:06 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_ni.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_ni.c,v 1.4 2020/07/25 22:29:06 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_ni.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_ni.c,v 1.4 2020/07/25 22:29:06 riastradh Exp $");
#ifdef _KERNEL
#include <sys/types.h>
@@ -204,6 +204,48 @@ aesni_xts_dec_impl(const struct aesdec *
fpu_kern_leave();
}
+static void
+aesni_cbcmac_update1_impl(const struct aesenc *enc,
+ const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
+ uint32_t nrounds)
+{
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ fpu_kern_enter();
+ aesni_cbcmac_update1(enc, in, nbytes, auth, nrounds);
+ fpu_kern_leave();
+}
+
+static void
+aesni_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ fpu_kern_enter();
+ aesni_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
+ fpu_kern_leave();
+}
+
+static void
+aesni_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ fpu_kern_enter();
+ aesni_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
+ fpu_kern_leave();
+}
+
static int
aesni_xts_update_selftest(void)
{
@@ -273,4 +315,7 @@ struct aes_impl aes_ni_impl = {
.ai_cbc_dec = aesni_cbc_dec_impl,
.ai_xts_enc = aesni_xts_enc_impl,
.ai_xts_dec = aesni_xts_dec_impl,
+ .ai_cbcmac_update1 = aesni_cbcmac_update1_impl,
+ .ai_ccm_enc1 = aesni_ccm_enc1_impl,
+ .ai_ccm_dec1 = aesni_ccm_dec1_impl,
};
Index: src/sys/crypto/aes/arch/x86/aes_ni_64.S
diff -u src/sys/crypto/aes/arch/x86/aes_ni_64.S:1.3 src/sys/crypto/aes/arch/x86/aes_ni_64.S:1.4
--- src/sys/crypto/aes/arch/x86/aes_ni_64.S:1.3 Sat Jul 25 22:11:05 2020
+++ src/sys/crypto/aes/arch/x86/aes_ni_64.S Sat Jul 25 22:29:06 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_ni_64.S,v 1.3 2020/07/25 22:11:05 riastradh Exp $ */
+/* $NetBSD: aes_ni_64.S,v 1.4 2020/07/25 22:29:06 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -951,6 +951,142 @@ ENTRY(aesni_xts_update)
END(aesni_xts_update)
/*
+ * aesni_cbcmac_update1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
+ * size_t nbytes@rdx, uint8_t auth[16] @rcx, uint32_t nrounds@r8d)
+ *
+ * Update CBC-MAC.
+ *
+ * nbytes must be a positive integral multiple of 16.
+ *
+ * Standard ABI calling convention.
+ */
+ENTRY(aesni_cbcmac_update1)
+ movdqu (%rcx),%xmm0 /* xmm0 := auth */
+ mov %rdx,%r10 /* r10 := nbytes */
+ mov %rcx,%rdx /* rdx := &auth */
+1: pxor (%rsi),%xmm0 /* xmm0 ^= plaintext block */
+ lea 0x10(%rsi),%rsi
+ mov %r8d,%ecx /* ecx := nrounds */
+ call aesni_enc1 /* xmm0 := auth'; trash rax,rcx,xmm8 */
+ sub $0x10,%r10
+ jnz 1b
+ movdqu %xmm0,(%rdx) /* store auth' */
+ ret
+END(aesni_cbcmac_update1)
+
+/*
+ * aesni_ccm_enc1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
+ * uint8_t *out@rdx, size_t nbytes@rcx,
+ * uint8_t authctr[32] @r8, uint32_t nrounds@r9d)
+ *
+ * Update CCM encryption.
+ *
+ * nbytes must be a positive integral multiple of 16.
+ *
+ * Standard ABI calling convention.
+ */
+ENTRY(aesni_ccm_enc1)
+ mov %rcx,%r10 /* r10 := nbytes */
+ movdqu 0x10(%r8),%xmm2 /* xmm2 := ctr (be) */
+ movdqa bswap32(%rip),%xmm4 /* xmm4 := bswap32 table */
+ movdqa ctr32_inc(%rip),%xmm5 /* xmm5 := (0,0,0,1) (le) */
+ movdqu (%r8),%xmm0 /* xmm0 := auth */
+ pshufb %xmm4,%xmm2 /* xmm2 := ctr (le) */
+1: movdqu (%rsi),%xmm3 /* xmm3 := plaintext block */
+ paddd %xmm5,%xmm2 /* increment ctr (32-bit) */
+ lea 0x10(%rsi),%rsi
+ movdqa %xmm2,%xmm1 /* xmm1 := ctr (le) */
+ mov %r9d,%ecx /* ecx := nrounds */
+ pshufb %xmm4,%xmm1 /* xmm1 := ctr (be) */
+ pxor %xmm3,%xmm0 /* xmm0 := auth ^ ptxt */
+ call aesni_enc2 /* trash rax/rcx/xmm8 */
+ pxor %xmm1,%xmm3 /* xmm3 := ciphertext block */
+ sub $0x10,%r10 /* count down bytes */
+ movdqu %xmm3,(%rdx) /* store ciphertext block */
+ lea 0x10(%rdx),%rdx
+ jnz 1b /* repeat if more blocks */
+ pshufb %xmm4,%xmm2 /* xmm2 := ctr (be) */
+ movdqu %xmm0,(%r8) /* store updated auth */
+ movdqu %xmm2,0x10(%r8) /* store updated ctr */
+ ret
+END(aesni_ccm_enc1)
+
+/*
+ * aesni_ccm_dec1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
+ * uint8_t *out@rdx, size_t nbytes@rcx,
+ * uint8_t authctr[32] @r8, uint32_t nrounds@r9d)
+ *
+ * Update CCM decryption.
+ *
+ * nbytes must be a positive integral multiple of 16.
+ *
+ * Standard ABI calling convention.
+ */
+ENTRY(aesni_ccm_dec1)
+ movdqu 0x10(%r8),%xmm2 /* xmm2 := ctr (be) */
+ movdqa bswap32(%rip),%xmm4 /* xmm4 := bswap32 table */
+ movdqa ctr32_inc(%rip),%xmm5 /* xmm5 := (0,0,0,1) (le) */
+ movdqu (%r8),%xmm1 /* xmm1 := auth */
+ pshufb %xmm4,%xmm2 /* xmm2 := ctr (le) */
+ mov %rcx,%r10 /* r10 := nbytes */
+
+ /* Decrypt the first block. */
+ paddd %xmm5,%xmm2 /* increment ctr (32-bit) */
+ mov %r9d,%ecx /* ecx := nrounds */
+ movdqa %xmm2,%xmm0 /* xmm0 := ctr (le) */
+ movdqu (%rsi),%xmm3 /* xmm3 := ctxt */
+ pshufb %xmm4,%xmm0 /* xmm0 := ctr (be) */
+ lea 0x10(%rsi),%rsi
+ call aesni_enc1 /* xmm0 := pad; trash rax/rcx/xmm8 */
+ jmp 2f
+
+1: /*
+ * Authenticate the last block and decrypt the next block
+ * simultaneously.
+ *
+ * xmm1 = auth ^ ptxt[-1]
+ * xmm2 = ctr[-1] (le)
+ */
+ paddd %xmm5,%xmm2 /* increment ctr (32-bit) */
+ mov %r9d,%ecx /* ecx := nrounds */
+ movdqa %xmm2,%xmm0 /* xmm0 := ctr (le) */
+ movdqu (%rsi),%xmm3 /* xmm3 := ctxt */
+ pshufb %xmm4,%xmm0 /* xmm0 := ctr (be) */
+ lea 0x10(%rsi),%rsi
+ call aesni_enc2 /* xmm0 := pad, xmm1 := auth';
+ * trash rax/rcx/xmm8 */
+2: pxor %xmm0,%xmm3 /* xmm3 := ptxt */
+ sub $0x10,%r10
+ movdqu %xmm3,(%rdx) /* store plaintext */
+ lea 0x10(%rdx),%rdx
+ pxor %xmm3,%xmm1 /* xmm1 := auth ^ ptxt */
+ jnz 1b
+
+ /* Authenticate the last block. */
+ movdqa %xmm1,%xmm0 /* xmm0 := auth ^ ptxt */
+ mov %r9d,%ecx /* ecx := nrounds */
+ call aesni_enc1 /* xmm0 := auth' */
+ pshufb %xmm4,%xmm2 /* xmm2 := ctr (be) */
+ movdqu %xmm0,(%r8) /* store updated auth */
+ movdqu %xmm2,0x10(%r8) /* store updated ctr */
+ ret
+END(aesni_ccm_dec1)
+
+ .section .rodata
+ .p2align 4
+ .type bswap32,@object
+bswap32:
+ .byte 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12
+END(bswap32)
+
+ .section .rodata
+ .p2align 4
+ .type ctr32_inc,@object
+ctr32_inc:
+ .byte 0,0,0,0, 0,0,0,0, 0,0,0,0, 1,0,0,0
+END(ctr32_inc)
+
+/*
* aesni_enc1(const struct aesenc *enckey@rdi, uint128_t block@xmm0,
* uint32_t nrounds@ecx)
*
@@ -976,6 +1112,35 @@ aesni_enc1:
END(aesni_enc1)
/*
+ * aesni_enc2(const struct aesenc *enckey@rdi, uint128_t block0@xmm0,
+ * uint128_t block1@xmm1, uint32_t nrounds@ecx)
+ *
+ * Encrypt two AES blocks in %xmm0 and %xmm1.
+ *
+ * Internal ABI. Uses %rax and %xmm8 as temporaries. Destroys %ecx.
+ */
+ .text
+ _ALIGN_TEXT
+ .type aesni_enc2,@function
+aesni_enc2:
+ movdqa (%rdi),%xmm8 /* xmm8 := first round key */
+ shl $4,%ecx /* ecx := total byte size of round keys */
+ lea 0x10(%rdi,%rcx),%rax /* rax := end of round key array */
+ neg %rcx /* rcx := byte offset of round key from end */
+ pxor %xmm8,%xmm0 /* xor in first round key */
+ pxor %xmm8,%xmm1
+ jmp 2f
+1: aesenc %xmm8,%xmm0
+ aesenc %xmm8,%xmm1
+2: movdqa (%rax,%rcx),%xmm8 /* load round key */
+ add $0x10,%rcx
+ jnz 1b /* repeat if there's more */
+ aesenclast %xmm8,%xmm0
+ aesenclast %xmm8,%xmm1
+ ret
+END(aesni_enc2)
+
+/*
* aesni_enc8(const struct aesenc *enckey@rdi, uint128_t block0@xmm0, ...,
* block7@xmm7, uint32_t nrounds@ecx)
*
Index: src/sys/crypto/aes/arch/x86/aes_ni.h
diff -u src/sys/crypto/aes/arch/x86/aes_ni.h:1.2 src/sys/crypto/aes/arch/x86/aes_ni.h:1.3
--- src/sys/crypto/aes/arch/x86/aes_ni.h:1.2 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_ni.h Sat Jul 25 22:29:06 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_ni.h,v 1.2 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_ni.h,v 1.3 2020/07/25 22:29:06 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -66,6 +66,13 @@ void aesni_xts_dec8(const struct aesdec
uint8_t[static 128], size_t, uint8_t[static 16], uint32_t);
void aesni_xts_update(const uint8_t[static 16], uint8_t[static 16]);
+void aesni_cbcmac_update1(const struct aesenc *, const uint8_t[static 16],
+ size_t, uint8_t[static 16], uint32_t);
+void aesni_ccm_enc1(const struct aesenc *, const uint8_t[static 16],
+ uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+void aesni_ccm_dec1(const struct aesenc *, const uint8_t[static 16],
+ uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+
extern struct aes_impl aes_ni_impl;
#endif /* _CRYPTO_AES_ARCH_X86_AES_NI_H */
