Author: jhb Date: Wed Dec 18 01:37:00 2019 New Revision: 355873 URL: https://svnweb.freebsd.org/changeset/base/355873
Log: Add support for TLS 1.3 using AES-GCM to the OCF backend for KTLS. Reviewed by: gallatin Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D22802 Modified: head/sys/opencrypto/ktls_ocf.c Modified: head/sys/opencrypto/ktls_ocf.c ============================================================================== --- head/sys/opencrypto/ktls_ocf.c Wed Dec 18 01:33:20 2019 (r355872) +++ head/sys/opencrypto/ktls_ocf.c Wed Dec 18 01:37:00 2019 (r355873) @@ -60,13 +60,21 @@ static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS SYSCTL_DECL(_kern_ipc_tls); SYSCTL_DECL(_kern_ipc_tls_stats); -static counter_u64_t ocf_gcm_crypts; -SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_gcm_crypts, CTLFLAG_RD, - &ocf_gcm_crypts, - "Total number of OCF GCM encryption operations"); +static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, CTLFLAG_RD, 0, + "Kernel TLS offload via OCF stats"); +static counter_u64_t ocf_tls12_gcm_crypts; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, + CTLFLAG_RD, &ocf_tls12_gcm_crypts, + "Total number of OCF TLS 1.2 GCM encryption operations"); + +static counter_u64_t ocf_tls13_gcm_crypts; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, + CTLFLAG_RD, &ocf_tls13_gcm_crypts, + "Total number of OCF TLS 1.3 GCM encryption operations"); + static counter_u64_t ocf_retries; -SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_retries, CTLFLAG_RD, +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, &ocf_retries, "Number of OCF encryption operation retries"); @@ -84,9 +92,10 @@ ktls_ocf_callback(struct cryptop *crp) } static int -ktls_ocf_encrypt(struct ktls_session *tls, const struct tls_record_layer *hdr, - uint8_t *trailer, struct iovec *iniov, struct iovec *outiov, int iovcnt, - uint64_t seqno, uint8_t record_type __unused) +ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, + struct iovec *outiov, int iovcnt, uint64_t seqno, + uint8_t record_type __unused) { struct uio uio; struct tls_aead_data ad; @@ -127,7 +136,7 @@ ktls_ocf_encrypt(struct ktls_session *tls, const struc iov[0].iov_base = &ad; iov[0].iov_len = sizeof(ad); uio.uio_resid = sizeof(ad); - + /* * OCF always does encryption in place, so copy the data if * needed. Ugh. @@ -171,7 +180,7 @@ ktls_ocf_encrypt(struct ktls_session *tls, const struc crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; memcpy(crde->crd_iv, &nd, sizeof(nd)); - counter_u64_add(ocf_gcm_crypts, 1); + counter_u64_add(ocf_tls12_gcm_crypts, 1); for (;;) { error = crypto_dispatch(crp); if (error) @@ -198,6 +207,118 @@ ktls_ocf_encrypt(struct ktls_session *tls, const struc return (error); } +static int +ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, + struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) +{ + struct uio uio; + struct tls_aead_data_13 ad; + char nonce[12]; + struct cryptodesc *crde, *crda; + struct cryptop *crp; + struct ocf_session *os; + struct ocf_operation *oo; + struct iovec *iov; + int i, error; + + os = tls->cipher; + + oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, + M_WAITOK | M_ZERO); + oo->os = os; + iov = oo->iov; + + crp = crypto_getreq(2); + if (crp == NULL) { + free(oo, M_KTLS_OCF); + return (ENOMEM); + } + + /* Setup the nonce. */ + memcpy(nonce, tls->params.iv, tls->params.iv_len); + *(uint64_t *)(nonce + 4) ^= htobe64(seqno); + + /* Setup the AAD. */ + ad.type = hdr->tls_type; + ad.tls_vmajor = hdr->tls_vmajor; + ad.tls_vminor = hdr->tls_vminor; + ad.tls_length = hdr->tls_length; + iov[0].iov_base = &ad; + iov[0].iov_len = sizeof(ad); + uio.uio_resid = sizeof(ad); + + /* + * OCF always does encryption in place, so copy the data if + * needed. Ugh. + */ + for (i = 0; i < iovcnt; i++) { + iov[i + 1] = outiov[i]; + if (iniov[i].iov_base != outiov[i].iov_base) + memcpy(outiov[i].iov_base, iniov[i].iov_base, + outiov[i].iov_len); + uio.uio_resid += outiov[i].iov_len; + } + + trailer[0] = record_type; + iov[iovcnt + 1].iov_base = trailer; + iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1; + uio.uio_resid += AES_GMAC_HASH_LEN + 1; + + uio.uio_iov = iov; + uio.uio_iovcnt = iovcnt + 2; + uio.uio_offset = 0; + uio.uio_segflg = UIO_SYSSPACE; + uio.uio_td = curthread; + + crp->crp_session = os->sid; + crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; + crp->crp_uio = &uio; + crp->crp_ilen = uio.uio_resid; + crp->crp_opaque = oo; + crp->crp_callback = ktls_ocf_callback; + + crde = crp->crp_desc; + crda = crde->crd_next; + + crda->crd_alg = os->crda_alg; + crda->crd_skip = 0; + crda->crd_len = sizeof(ad); + crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; + + crde->crd_alg = CRYPTO_AES_NIST_GCM_16; + crde->crd_skip = sizeof(ad); + crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); + crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; + memcpy(crde->crd_iv, nonce, sizeof(nonce)); + + counter_u64_add(ocf_tls13_gcm_crypts, 1); + for (;;) { + error = crypto_dispatch(crp); + if (error) + break; + + mtx_lock(&os->lock); + while (!oo->done) + mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); + mtx_unlock(&os->lock); + + if (crp->crp_etype != EAGAIN) { + error = crp->crp_etype; + break; + } + + crp->crp_etype = 0; + crp->crp_flags &= ~CRYPTO_F_DONE; + oo->done = false; + counter_u64_add(ocf_retries, 1); + } + + crypto_freereq(crp); + free(oo, M_KTLS_OCF); + return (error); +} + static void ktls_ocf_free(struct ktls_session *tls) { @@ -221,8 +342,6 @@ ktls_ocf_try(struct socket *so, struct ktls_session *t switch (tls->params.cipher_algorithm) { case CRYPTO_AES_NIST_GCM_16: - if (tls->params.iv_len != TLS_AEAD_GCM_LEN) - return (EINVAL); switch (tls->params.cipher_key_len) { case 128 / 8: cria.cri_alg = CRYPTO_AES_128_NIST_GMAC; @@ -240,10 +359,10 @@ ktls_ocf_try(struct socket *so, struct ktls_session *t return (EPROTONOSUPPORT); } - /* Only TLS 1.1 and TLS 1.2 are currently supported. */ + /* Only TLS 1.2 and 1.3 are supported. */ if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || - tls->params.tls_vminor < TLS_MINOR_VER_ONE || - tls->params.tls_vminor > TLS_MINOR_VER_TWO) + tls->params.tls_vminor < TLS_MINOR_VER_TWO || + tls->params.tls_vminor > TLS_MINOR_VER_THREE) return (EPROTONOSUPPORT); os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); @@ -265,7 +384,10 @@ ktls_ocf_try(struct socket *so, struct ktls_session *t os->crda_alg = cria.cri_alg; mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); tls->cipher = os; - tls->sw_encrypt = ktls_ocf_encrypt; + if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) + tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; + else + tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; tls->free = ktls_ocf_free; return (0); } @@ -284,14 +406,16 @@ ktls_ocf_modevent(module_t mod, int what, void *arg) switch (what) { case MOD_LOAD: - ocf_gcm_crypts = counter_u64_alloc(M_WAITOK); + ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); + ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); ocf_retries = counter_u64_alloc(M_WAITOK); return (ktls_crypto_backend_register(&ocf_backend)); case MOD_UNLOAD: error = ktls_crypto_backend_deregister(&ocf_backend); if (error) return (error); - counter_u64_free(ocf_gcm_crypts); + counter_u64_free(ocf_tls12_gcm_crypts); + counter_u64_free(ocf_tls13_gcm_crypts); counter_u64_free(ocf_retries); return (0); default: _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
