Author: gallatin
Date: Thu Apr 25 15:37:28 2019
New Revision: 346677
URL: https://svnweb.freebsd.org/changeset/base/346677
Log:
Track TCP connection's NUMA domain in the inpcb
Drivers can now pass up numa domain information via the
mbuf numa domain field. This information is then used
by TCP syncache_socket() to associate that information
with the inpcb. The domain information is then fed back
into transmitted mbufs in ip{6}_output(). This mechanism
is nearly identical to what is done to track RSS hash values
in the inp_flowid.
Follow on changes will use this information for lacp egress
port selection, binding TCP pacers to the appropriate NUMA
domain, etc.
Reviewed by: markj, kib, slavash, bz, scottl, jtl, tuexen
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D20028
Modified:
head/sys/dev/cxgbe/t4_sge.c
head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
head/sys/kern/uipc_mbuf.c
head/sys/netinet/in_pcb.c
head/sys/netinet/in_pcb.h
head/sys/netinet/ip_output.c
head/sys/netinet/tcp_syncache.c
head/sys/netinet6/ip6_output.c
Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c Thu Apr 25 15:31:35 2019 (r346676)
+++ head/sys/dev/cxgbe/t4_sge.c Thu Apr 25 15:37:28 2019 (r346677)
@@ -2046,6 +2046,9 @@ t4_eth_rx(struct sge_iq *iq, const struct rss_header *
rxq->vlan_extraction++;
}
+#ifdef NUMA
+ m0->m_pkthdr.numa_domain = ifp->if_numa_domain;
+#endif
#if defined(INET) || defined(INET6)
if (iq->flags & IQ_LRO_ENABLED) {
if (sort_before_lro(lro)) {
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c Thu Apr 25 15:31:35 2019
(r346676)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c Thu Apr 25 15:37:28 2019
(r346677)
@@ -520,6 +520,9 @@ rx_common:
mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
rq->stats.bytes += byte_cnt;
rq->stats.packets++;
+#ifdef NUMA
+ mb->m_pkthdr.numa_domain = rq->ifp->if_numa_domain;
+#endif
#if !defined(HAVE_TCP_LRO_RX)
tcp_lro_queue_mbuf(&rq->lro, mb);
Modified: head/sys/kern/uipc_mbuf.c
==============================================================================
--- head/sys/kern/uipc_mbuf.c Thu Apr 25 15:31:35 2019 (r346676)
+++ head/sys/kern/uipc_mbuf.c Thu Apr 25 15:37:28 2019 (r346677)
@@ -341,6 +341,9 @@ m_pkthdr_init(struct mbuf *m, int how)
#endif
m->m_data = m->m_pktdat;
bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
+#ifdef NUMA
+ m->m_pkthdr.numa_domain = M_NODOM;
+#endif
#ifdef MAC
/* If the label init fails, fail the alloc */
error = mac_mbuf_init(m, how);
Modified: head/sys/netinet/in_pcb.c
==============================================================================
--- head/sys/netinet/in_pcb.c Thu Apr 25 15:31:35 2019 (r346676)
+++ head/sys/netinet/in_pcb.c Thu Apr 25 15:37:28 2019 (r346677)
@@ -510,6 +510,9 @@ in_pcballoc(struct socket *so, struct inpcbinfo *pcbin
if (inp == NULL)
return (ENOBUFS);
bzero(&inp->inp_start_zero, inp_zero_size);
+#ifdef NUMA
+ inp->inp_numa_domain = M_NODOM;
+#endif
inp->inp_pcbinfo = pcbinfo;
inp->inp_socket = so;
inp->inp_cred = crhold(so->so_cred);
Modified: head/sys/netinet/in_pcb.h
==============================================================================
--- head/sys/netinet/in_pcb.h Thu Apr 25 15:31:35 2019 (r346676)
+++ head/sys/netinet/in_pcb.h Thu Apr 25 15:37:28 2019 (r346677)
@@ -272,7 +272,7 @@ struct inpcb {
inp_hpts_calls :1, /* (i) from output hpts */
inp_input_calls :1, /* (i) from input hpts */
inp_spare_bits2 : 4;
- uint8_t inp_spare_byte; /* Compiler hole */
+ uint8_t inp_numa_domain; /* numa domain */
void *inp_ppcb; /* (i) pointer to per-protocol pcb */
struct socket *inp_socket; /* (i) back pointer to socket */
uint32_t inp_hptsslot; /* Hpts wheel slot this tcb is
Lock(i&b) */
Modified: head/sys/netinet/ip_output.c
==============================================================================
--- head/sys/netinet/ip_output.c Thu Apr 25 15:31:35 2019
(r346676)
+++ head/sys/netinet/ip_output.c Thu Apr 25 15:37:28 2019
(r346677)
@@ -247,6 +247,9 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct rou
m->m_pkthdr.flowid = inp->inp_flowid;
M_HASHTYPE_SET(m, inp->inp_flowtype);
}
+#ifdef NUMA
+ m->m_pkthdr.numa_domain = inp->inp_numa_domain;
+#endif
}
if (ro == NULL) {
Modified: head/sys/netinet/tcp_syncache.c
==============================================================================
--- head/sys/netinet/tcp_syncache.c Thu Apr 25 15:31:35 2019
(r346676)
+++ head/sys/netinet/tcp_syncache.c Thu Apr 25 15:37:28 2019
(r346677)
@@ -777,6 +777,9 @@ syncache_socket(struct syncache *sc, struct socket *ls
if (m != NULL && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
inp->inp_flowid = m->m_pkthdr.flowid;
inp->inp_flowtype = M_HASHTYPE_GET(m);
+#ifdef NUMA
+ inp->inp_numa_domain = m->m_pkthdr.numa_domain;
+#endif
}
/*
Modified: head/sys/netinet6/ip6_output.c
==============================================================================
--- head/sys/netinet6/ip6_output.c Thu Apr 25 15:31:35 2019
(r346676)
+++ head/sys/netinet6/ip6_output.c Thu Apr 25 15:37:28 2019
(r346677)
@@ -322,6 +322,9 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt,
m->m_pkthdr.flowid = inp->inp_flowid;
M_HASHTYPE_SET(m, inp->inp_flowtype);
}
+#ifdef NUMA
+ m->m_pkthdr.numa_domain = inp->inp_numa_domain;
+#endif
}
#if defined(IPSEC) || defined(IPSEC_SUPPORT)
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
