Module Name: src Committed By: ozaki-r Date: Thu Jul 20 08:07:14 UTC 2017
Modified Files: src/sys/netipsec: xform_ah.c xform_esp.c xform_ipcomp.c Log Message: Use pool to allocate tdb_crypto For ESP and AH, we need to allocate an extra variable space in addition to struct tdb_crypto. The fixed size of pool items may be larger than an actual requisite size of a buffer, but still the performance improvement by replacing malloc with pool wins. To generate a diff of this commit: cvs rdiff -u -r1.67 -r1.68 src/sys/netipsec/xform_ah.c cvs rdiff -u -r1.65 -r1.66 src/sys/netipsec/xform_esp.c cvs rdiff -u -r1.46 -r1.47 src/sys/netipsec/xform_ipcomp.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/netipsec/xform_ah.c diff -u src/sys/netipsec/xform_ah.c:1.67 src/sys/netipsec/xform_ah.c:1.68 --- src/sys/netipsec/xform_ah.c:1.67 Thu Jul 20 03:17:59 2017 +++ src/sys/netipsec/xform_ah.c Thu Jul 20 08:07:14 2017 @@ -1,4 +1,4 @@ -/* $NetBSD: xform_ah.c,v 1.67 2017/07/20 03:17:59 ozaki-r Exp $ */ +/* $NetBSD: xform_ah.c,v 1.68 2017/07/20 08:07:14 ozaki-r Exp $ */ /* $FreeBSD: src/sys/netipsec/xform_ah.c,v 1.1.4.1 2003/01/24 05:11:36 sam Exp $ */ /* $OpenBSD: ip_ah.c,v 1.63 2001/06/26 06:18:58 angelos Exp $ */ /* @@ -39,7 +39,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: xform_ah.c,v 1.67 2017/07/20 03:17:59 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: xform_ah.c,v 1.68 2017/07/20 08:07:14 ozaki-r Exp $"); #if defined(_KERNEL_OPT) #include "opt_inet.h" @@ -54,6 +54,7 @@ __KERNEL_RCSID(0, "$NetBSD: xform_ah.c,v #include <sys/kernel.h> #include <sys/sysctl.h> #include <sys/socketvar.h> /* for softnet_lock */ +#include <sys/pool.h> #include <net/if.h> @@ -61,6 +62,7 @@ __KERNEL_RCSID(0, "$NetBSD: xform_ah.c,v #include <netinet/in_systm.h> #include <netinet/ip.h> #include <netinet/ip_ecn.h> +#include <netinet/ip_var.h> #include <netinet/ip6.h> #include <net/route.h> @@ -114,13 +116,16 @@ SYSCTL_STRUCT(_net_inet_ah, IPSECCTL_STA static unsigned char ipseczeroes[256]; /* larger than an ip6 extension hdr */ -static int ah_max_authsize; /* max authsize over all algorithms */ +int ah_max_authsize; /* max authsize over all algorithms */ static int ah_input_cb(struct cryptop *); static int ah_output_cb(struct cryptop *); const uint8_t ah_stats[256] = { SADB_AALG_STATS_INIT }; +static struct pool ah_tdb_crypto_pool; +static size_t ah_pool_item_size; + /* * NB: this is public for use by the PF_KEY support. */ @@ -695,7 +700,9 @@ ah_input(struct mbuf *m, struct secasvar size_t extra = skip + rplen + authsize; size += extra; - tc = malloc(size, M_XDATA, M_NOWAIT|M_ZERO); + KASSERTMSG(size <= ah_pool_item_size, + "size=%zu > ah_pool_item_size=%zu\n", size, ah_pool_item_size); + tc = pool_get(&ah_tdb_crypto_pool, PR_NOWAIT); if (tc == NULL) { DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); stat = AH_STAT_CRYPTO; @@ -753,7 +760,7 @@ ah_input(struct mbuf *m, struct secasvar bad: if (tc != NULL) - free(tc, M_XDATA); + pool_put(&ah_tdb_crypto_pool, tc); if (crp != NULL) crypto_freereq(crp); if (m != NULL) @@ -888,7 +895,8 @@ ah_input_cb(struct cryptop *crp) /* Copyback the saved (uncooked) network headers. */ m_copyback(m, 0, skip, ptr); - free(tc, M_XDATA), tc = NULL; /* No longer needed */ + pool_put(&ah_tdb_crypto_pool, tc); + tc = NULL; /* * Header is now authenticated. @@ -937,7 +945,7 @@ bad: if (m != NULL) m_freem(m); if (tc != NULL) - free(tc, M_XDATA); + pool_put(&ah_tdb_crypto_pool, tc); if (crp != NULL) crypto_freereq(crp); return error; @@ -1097,7 +1105,7 @@ ah_output( crda->crd_klen = _KEYBITS(sav->key_auth); /* Allocate IPsec-specific opaque crypto info. */ - tc = malloc(sizeof(*tc) + skip, M_XDATA, M_NOWAIT|M_ZERO); + tc = pool_get(&ah_tdb_crypto_pool, PR_NOWAIT); if (tc == NULL) { crypto_freereq(crp); DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); @@ -1131,7 +1139,7 @@ ah_output( skip, ahx->type, 1); if (error != 0) { m = NULL; /* mbuf was free'd by ah_massage_headers. */ - free(tc, M_XDATA); + pool_put(&ah_tdb_crypto_pool, tc); crypto_freereq(crp); goto bad; } @@ -1232,7 +1240,7 @@ ah_output_cb(struct cryptop *crp) m_copyback(m, 0, skip, ptr); /* No longer needed. */ - free(tc, M_XDATA); + pool_put(&ah_tdb_crypto_pool, tc); crypto_freereq(crp); #ifdef IPSEC_DEBUG @@ -1264,7 +1272,7 @@ bad: splx(s); if (m) m_freem(m); - free(tc, M_XDATA); + pool_put(&ah_tdb_crypto_pool, tc); crypto_freereq(crp); return error; } @@ -1312,5 +1320,11 @@ ah_attach(void) #undef MAXAUTHSIZE + ah_pool_item_size = sizeof(struct tdb_crypto) + + sizeof(struct ip) + MAX_IPOPTLEN + + sizeof(struct ah) + sizeof(uint32_t) + ah_max_authsize; + pool_init(&ah_tdb_crypto_pool, ah_pool_item_size, + 0, 0, 0, "ah_tdb_crypto", NULL, IPL_SOFTNET); + xform_register(&ah_xformsw); } Index: src/sys/netipsec/xform_esp.c diff -u src/sys/netipsec/xform_esp.c:1.65 src/sys/netipsec/xform_esp.c:1.66 --- src/sys/netipsec/xform_esp.c:1.65 Wed Jul 19 10:26:09 2017 +++ src/sys/netipsec/xform_esp.c Thu Jul 20 08:07:14 2017 @@ -1,4 +1,4 @@ -/* $NetBSD: xform_esp.c,v 1.65 2017/07/19 10:26:09 ozaki-r Exp $ */ +/* $NetBSD: xform_esp.c,v 1.66 2017/07/20 08:07:14 ozaki-r Exp $ */ /* $FreeBSD: src/sys/netipsec/xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */ /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ @@ -39,7 +39,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.65 2017/07/19 10:26:09 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.66 2017/07/20 08:07:14 ozaki-r Exp $"); #if defined(_KERNEL_OPT) #include "opt_inet.h" @@ -55,6 +55,7 @@ __KERNEL_RCSID(0, "$NetBSD: xform_esp.c, #include <sys/sysctl.h> #include <sys/socketvar.h> /* for softnet_lock */ #include <sys/cprng.h> +#include <sys/pool.h> #include <net/if.h> @@ -103,6 +104,9 @@ static int esp_output_cb(struct cryptop const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT }; +static struct pool esp_tdb_crypto_pool; +static size_t esp_pool_item_size; + /* * NB: this is public for use by the PF_KEY support. * NB: if you add support here; be sure to add code to esp_attach below! @@ -375,8 +379,11 @@ esp_input(struct mbuf *m, struct secasva } /* Get IPsec-specific opaque pointer */ - size_t extra = esph == NULL ? 0 : alen; - tc = malloc(sizeof(*tc) + extra, M_XDATA, M_NOWAIT|M_ZERO); + size_t extra __diagused = esph == NULL ? 0 : alen; + KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size, + "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n", + sizeof(*tc) + extra, esp_pool_item_size); + tc = pool_get(&esp_tdb_crypto_pool, PR_NOWAIT); if (tc == NULL) { DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); error = ENOBUFS; @@ -458,7 +465,7 @@ esp_input(struct mbuf *m, struct secasva return crypto_dispatch(crp); out2: - free(tc, M_XDATA); + pool_put(&esp_tdb_crypto_pool, tc); out1: crypto_freereq(crp); out: @@ -586,7 +593,8 @@ esp_input_cb(struct cryptop *crp) } /* Release the crypto descriptors */ - free(tc, M_XDATA), tc = NULL; + pool_put(&esp_tdb_crypto_pool, tc); + tc = NULL; crypto_freereq(crp), crp = NULL; /* @@ -678,7 +686,7 @@ bad: if (m != NULL) m_freem(m); if (tc != NULL) - free(tc, M_XDATA); + pool_put(&esp_tdb_crypto_pool, tc); if (crp != NULL) crypto_freereq(crp); return error; @@ -884,7 +892,7 @@ esp_output( crda = crp->crp_desc; /* IPsec-specific opaque crypto info. */ - tc = malloc(sizeof(*tc), M_XDATA, M_NOWAIT|M_ZERO); + tc = pool_get(&esp_tdb_crypto_pool, PR_NOWAIT); if (tc == NULL) { crypto_freereq(crp); DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); @@ -1005,7 +1013,7 @@ esp_output_cb(struct cryptop *crp) AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); /* Release crypto descriptors. */ - free(tc, M_XDATA); + pool_put(&esp_tdb_crypto_pool, tc); crypto_freereq(crp); #ifdef IPSEC_DEBUG @@ -1041,7 +1049,7 @@ bad: splx(s); if (m) m_freem(m); - free(tc, M_XDATA); + pool_put(&esp_tdb_crypto_pool, tc); crypto_freereq(crp); return error; } @@ -1063,6 +1071,12 @@ esp_attach(void) espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS); + extern int ah_max_authsize; + KASSERT(ah_max_authsize != 0); + esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize; + pool_init(&esp_tdb_crypto_pool, esp_pool_item_size, + 0, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET); + #define MAXIV(xform) \ if (xform.ivsize > esp_max_ivlen) \ esp_max_ivlen = xform.ivsize \ Index: src/sys/netipsec/xform_ipcomp.c diff -u src/sys/netipsec/xform_ipcomp.c:1.46 src/sys/netipsec/xform_ipcomp.c:1.47 --- src/sys/netipsec/xform_ipcomp.c:1.46 Wed Jul 19 10:26:09 2017 +++ src/sys/netipsec/xform_ipcomp.c Thu Jul 20 08:07:14 2017 @@ -1,4 +1,4 @@ -/* $NetBSD: xform_ipcomp.c,v 1.46 2017/07/19 10:26:09 ozaki-r Exp $ */ +/* $NetBSD: xform_ipcomp.c,v 1.47 2017/07/20 08:07:14 ozaki-r Exp $ */ /* $FreeBSD: src/sys/netipsec/xform_ipcomp.c,v 1.1.4.1 2003/01/24 05:11:36 sam Exp $ */ /* $OpenBSD: ip_ipcomp.c,v 1.1 2001/07/05 12:08:52 jjbg Exp $ */ @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: xform_ipcomp.c,v 1.46 2017/07/19 10:26:09 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: xform_ipcomp.c,v 1.47 2017/07/20 08:07:14 ozaki-r Exp $"); /* IP payload compression protocol (IPComp), see RFC 2393 */ #if defined(_KERNEL_OPT) @@ -45,6 +45,7 @@ __KERNEL_RCSID(0, "$NetBSD: xform_ipcomp #include <sys/protosw.h> #include <sys/sysctl.h> #include <sys/socketvar.h> /* for softnet_lock */ +#include <sys/pool.h> #include <netinet/in.h> #include <netinet/in_systm.h> @@ -88,6 +89,8 @@ static int ipcomp_output_cb(struct crypt const uint8_t ipcomp_stats[256] = { SADB_CALG_STATS_INIT }; +static struct pool ipcomp_tdb_crypto_pool; + const struct comp_algo * ipcomp_algorithm_lookup(int alg) { @@ -162,7 +165,7 @@ ipcomp_input(struct mbuf *m, struct seca return ENOBUFS; } /* Get IPsec-specific opaque pointer */ - tc = malloc(sizeof(*tc), M_XDATA, M_NOWAIT|M_ZERO); + tc = pool_get(&ipcomp_tdb_crypto_pool, PR_NOWAIT); if (tc == NULL) { m_freem(m); crypto_freereq(crp); @@ -175,7 +178,7 @@ ipcomp_input(struct mbuf *m, struct seca if (error) { DPRINTF(("%s: m_makewritable failed\n", __func__)); m_freem(m); - free(tc, M_XDATA); + pool_put(&ipcomp_tdb_crypto_pool, tc); crypto_freereq(crp); IPCOMP_STATINC(IPCOMP_STAT_CRYPTO); return error; @@ -300,7 +303,8 @@ ipcomp_input_cb(struct cryptop *crp) clen = crp->crp_olen; /* Length of data after processing */ /* Release the crypto descriptors */ - free(tc, M_XDATA), tc = NULL; + pool_put(&ipcomp_tdb_crypto_pool, tc); + tc = NULL; crypto_freereq(crp), crp = NULL; /* In case it's not done already, adjust the size of the mbuf chain */ @@ -357,7 +361,7 @@ bad: if (m) m_freem(m); if (tc != NULL) - free(tc, M_XDATA); + pool_put(&ipcomp_tdb_crypto_pool, tc); if (crp) crypto_freereq(crp); return error; @@ -470,7 +474,7 @@ ipcomp_output( crdc->crd_alg = ipcompx->type; /* IPsec-specific opaque crypto info */ - tc = malloc(sizeof(*tc), M_XDATA, M_NOWAIT|M_ZERO); + tc = pool_get(&ipcomp_tdb_crypto_pool, PR_NOWAIT); if (tc == NULL) { IPCOMP_STATINC(IPCOMP_STAT_CRYPTO); DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); @@ -641,7 +645,7 @@ ipcomp_output_cb(struct cryptop *crp) /* Release the crypto descriptor */ - free(tc, M_XDATA); + pool_put(&ipcomp_tdb_crypto_pool, tc); crypto_freereq(crp); /* NB: m is reclaimed by ipsec_process_done. */ @@ -659,7 +663,7 @@ bad: splx(s); if (m) m_freem(m); - free(tc, M_XDATA); + pool_put(&ipcomp_tdb_crypto_pool, tc); crypto_freereq(crp); return error; } @@ -679,5 +683,7 @@ void ipcomp_attach(void) { ipcompstat_percpu = percpu_alloc(sizeof(uint64_t) * IPCOMP_NSTATS); + pool_init(&ipcomp_tdb_crypto_pool, sizeof(struct tdb_crypto), + 0, 0, 0, "ipcomp_tdb_crypto", NULL, IPL_SOFTNET); xform_register(&ipcomp_xformsw); }