Module Name: src
Committed By: riastradh
Date: Sat Jul 25 22:31:04 UTC 2020
Modified Files:
src/sys/crypto/aes/arch/x86: aes_ssse3.h aes_ssse3_impl.c
aes_ssse3_subr.c immintrin.h
Log Message:
Implement AES-CCM with SSSE3.
To generate a diff of this commit:
cvs rdiff -u -r1.2 -r1.3 src/sys/crypto/aes/arch/x86/aes_ssse3.h \
src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c \
src/sys/crypto/aes/arch/x86/immintrin.h
cvs rdiff -u -r1.3 -r1.4 src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/crypto/aes/arch/x86/aes_ssse3.h
diff -u src/sys/crypto/aes/arch/x86/aes_ssse3.h:1.2 src/sys/crypto/aes/arch/x86/aes_ssse3.h:1.3
--- src/sys/crypto/aes/arch/x86/aes_ssse3.h:1.2 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_ssse3.h Sat Jul 25 22:31:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_ssse3.h,v 1.2 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_ssse3.h,v 1.3 2020/07/25 22:31:04 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -59,6 +59,12 @@ void aes_ssse3_xts_enc(const struct aese
uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
void aes_ssse3_xts_dec(const struct aesdec *, const uint8_t[static 16],
uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
+void aes_ssse3_cbcmac_update1(const struct aesenc *, const uint8_t[static 16],
+ size_t, uint8_t[static 16], uint32_t);
+void aes_ssse3_ccm_enc1(const struct aesenc *, const uint8_t[static 16],
+ uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+void aes_ssse3_ccm_dec1(const struct aesenc *, const uint8_t[static 16],
+ uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
int aes_ssse3_selftest(void);
Index: src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c
diff -u src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c:1.2 src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c:1.3
--- src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c:1.2 Tue Jun 30 20:32:11 2020
+++ src/sys/crypto/aes/arch/x86/aes_ssse3_subr.c Sat Jul 25 22:31:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_ssse3_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $ */
+/* $NetBSD: aes_ssse3_subr.c,v 1.3 2020/07/25 22:31:04 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_subr.c,v 1.3 2020/07/25 22:31:04 riastradh Exp $");
#ifdef _KERNEL
#include <sys/systm.h>
@@ -208,6 +208,75 @@ aes_ssse3_xts_dec(const struct aesdec *d
storeblock(tweak, t);
}
+void
+aes_ssse3_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
+ size_t nbytes, uint8_t auth0[static 16], uint32_t nrounds)
+{
+ __m128i auth;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ auth = loadblock(auth0);
+ for (; nbytes; nbytes -= 16, in += 16)
+ auth = aes_ssse3_enc1(enc, auth ^ loadblock(in), nrounds);
+ storeblock(auth0, auth);
+}
+
+void
+aes_ssse3_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+ const __m128i ctr32_inc = _mm_set_epi32(1, 0, 0, 0);
+ const __m128i bs32 =
+ _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203);
+ __m128i auth, ctr_be, ctr, ptxt;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ auth = loadblock(authctr);
+ ctr_be = loadblock(authctr + 16);
+ ctr = _mm_shuffle_epi8(ctr_be, bs32);
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ ptxt = loadblock(in);
+ auth = aes_ssse3_enc1(enc, auth ^ ptxt, nrounds);
+ ctr = _mm_add_epi32(ctr, ctr32_inc);
+ ctr_be = _mm_shuffle_epi8(ctr, bs32);
+ storeblock(out, ptxt ^ aes_ssse3_enc1(enc, ctr_be, nrounds));
+ }
+ storeblock(authctr, auth);
+ storeblock(authctr + 16, ctr_be);
+}
+
+void
+aes_ssse3_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+ const __m128i ctr32_inc = _mm_set_epi32(1, 0, 0, 0);
+ const __m128i bs32 =
+ _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203);
+ __m128i auth, ctr_be, ctr, ptxt;
+
+ KASSERT(nbytes);
+ KASSERT(nbytes % 16 == 0);
+
+ auth = loadblock(authctr);
+ ctr_be = loadblock(authctr + 16);
+ ctr = _mm_shuffle_epi8(ctr_be, bs32);
+ for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+ ctr = _mm_add_epi32(ctr, ctr32_inc);
+ ctr_be = _mm_shuffle_epi8(ctr, bs32);
+ ptxt = loadblock(in) ^ aes_ssse3_enc1(enc, ctr_be, nrounds);
+ storeblock(out, ptxt);
+ auth = aes_ssse3_enc1(enc, auth ^ ptxt, nrounds);
+ }
+ storeblock(authctr, auth);
+ storeblock(authctr + 16, ctr_be);
+}
+
int
aes_ssse3_selftest(void)
{
Index: src/sys/crypto/aes/arch/x86/immintrin.h
diff -u src/sys/crypto/aes/arch/x86/immintrin.h:1.2 src/sys/crypto/aes/arch/x86/immintrin.h:1.3
--- src/sys/crypto/aes/arch/x86/immintrin.h:1.2 Mon Jun 29 23:51:35 2020
+++ src/sys/crypto/aes/arch/x86/immintrin.h Sat Jul 25 22:31:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: immintrin.h,v 1.2 2020/06/29 23:51:35 riastradh Exp $ */
+/* $NetBSD: immintrin.h,v 1.3 2020/07/25 22:31:04 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -51,6 +51,7 @@ typedef long long __m128i_u
typedef long long __v2di __attribute__((__vector_size__(16)));
typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
typedef int __v4si __attribute__((__vector_size__(16)));
+typedef unsigned __v4su __attribute__((__vector_size__(16)));
typedef float __v4sf __attribute__((__vector_size__(16)));
typedef short __v8hi __attribute__((__vector_size__(16)));
typedef char __v16qi __attribute__((__vector_size__(16)));
@@ -65,6 +66,7 @@ typedef long long __m128i_u
typedef long long __v2di __attribute__((__vector_size__(16)));
typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
typedef int __v4si __attribute__((__vector_size__(16)));
+typedef unsigned __v4su __attribute__((__vector_size__(16)));
typedef float __v4sf __attribute__((__vector_size__(16)));
typedef short __v8hi __attribute__((__vector_size__(16)));
typedef char __v16qi __attribute__((__vector_size__(16)));
@@ -83,6 +85,13 @@ typedef char __v16qi __attribute__((__ve
#define _SSSE3_ATTR __attribute__((target("ssse3")))
+_INTRINSATTR
+static __inline __m128i
+_mm_add_epi32(__m128i __a, __m128i __b)
+{
+ return (__m128i)((__v4su)__a + (__v4su)__b);
+}
+
#if defined(__GNUC__) && !defined(__clang__)
#define _mm_alignr_epi8(hi,lo,bytes) \
(__m128i)__builtin_ia32_palignr128((__v2di)(__m128i)(hi), \
Index: src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c
diff -u src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c:1.3 src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c:1.4
--- src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c:1.3 Sat Jul 25 22:12:57 2020
+++ src/sys/crypto/aes/arch/x86/aes_ssse3_impl.c Sat Jul 25 22:31:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: aes_ssse3_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $ */
+/* $NetBSD: aes_ssse3_impl.c,v 1.4 2020/07/25 22:31:04 riastradh Exp $ */
/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_impl.c,v 1.4 2020/07/25 22:31:04 riastradh Exp $");
#include <crypto/aes/aes.h>
#include <crypto/aes/aes_impl.h>
@@ -136,6 +136,39 @@ aes_ssse3_xts_dec_impl(const struct aesd
fpu_kern_leave();
}
+static void
+aes_ssse3_cbcmac_update1_impl(const struct aesenc *enc,
+ const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
+ uint32_t nrounds)
+{
+
+ fpu_kern_enter();
+ aes_ssse3_cbcmac_update1(enc, in, nbytes, auth, nrounds);
+ fpu_kern_leave();
+}
+
+static void
+aes_ssse3_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+
+ fpu_kern_enter();
+ aes_ssse3_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
+ fpu_kern_leave();
+}
+
+static void
+aes_ssse3_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+ uint32_t nrounds)
+{
+
+ fpu_kern_enter();
+ aes_ssse3_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
+ fpu_kern_leave();
+}
+
static int
aes_ssse3_probe(void)
{
@@ -183,4 +216,7 @@ struct aes_impl aes_ssse3_impl = {
.ai_cbc_dec = aes_ssse3_cbc_dec_impl,
.ai_xts_enc = aes_ssse3_xts_enc_impl,
.ai_xts_dec = aes_ssse3_xts_dec_impl,
+ .ai_cbcmac_update1 = aes_ssse3_cbcmac_update1_impl,
+ .ai_ccm_enc1 = aes_ssse3_ccm_enc1_impl,
+ .ai_ccm_dec1 = aes_ssse3_ccm_dec1_impl,
};
