Module Name: src
Committed By: ozaki-r
Date: Wed Aug 31 09:14:47 UTC 2016
Modified Files:
src/sys/netinet: ip_input.c
src/sys/netinet6: ip6_forward.c ip6_input.c
Log Message:
Make ipforward_rt and ip6_forward_rt percpu
Sharing one rtcache between CPUs is just a bad idea.
Reviewed by knakahara@
To generate a diff of this commit:
cvs rdiff -u -r1.339 -r1.340 src/sys/netinet/ip_input.c
cvs rdiff -u -r1.80 -r1.81 src/sys/netinet6/ip6_forward.c
cvs rdiff -u -r1.166 -r1.167 src/sys/netinet6/ip6_input.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/netinet/ip_input.c
diff -u src/sys/netinet/ip_input.c:1.339 src/sys/netinet/ip_input.c:1.340
--- src/sys/netinet/ip_input.c:1.339 Mon Aug 1 03:15:30 2016
+++ src/sys/netinet/ip_input.c Wed Aug 31 09:14:47 2016
@@ -1,4 +1,4 @@
-/* $NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $ */
+/* $NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $ */
/*
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_inet.h"
@@ -225,7 +225,7 @@ pfil_head_t * inet_pfil_hook __read_mo
ipid_state_t * ip_ids __read_mostly;
percpu_t * ipstat_percpu __read_mostly;
-static struct route ipforward_rt __cacheline_aligned;
+static percpu_t *ipforward_rt_percpu __cacheline_aligned;
uint16_t ip_id;
@@ -345,6 +345,10 @@ ip_init(void)
#endif /* MBUFTRACE */
ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS);
+
+ ipforward_rt_percpu = percpu_alloc(sizeof(struct route));
+ if (ipforward_rt_percpu == NULL)
+ panic("failed to allocate ipforward_rt_percpu");
}
static struct in_ifaddr *
@@ -1163,11 +1167,14 @@ ip_rtaddr(struct in_addr dst)
struct sockaddr dst;
struct sockaddr_in dst4;
} u;
+ struct route *ro;
sockaddr_in_init(&u.dst4, &dst, 0);
SOFTNET_LOCK();
- rt = rtcache_lookup(&ipforward_rt, &u.dst);
+ ro = percpu_getref(ipforward_rt_percpu);
+ rt = rtcache_lookup(ro, &u.dst);
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
if (rt == NULL)
return NULL;
@@ -1300,6 +1307,7 @@ ip_forward(struct mbuf *m, int srcrt, st
struct sockaddr_in dst4;
} u;
uint64_t *ips;
+ struct route *ro;
KASSERTMSG(cpu_softintr_p(), "ip_forward: not in the software "
"interrupt handler; synchronization assumptions violated");
@@ -1331,7 +1339,9 @@ ip_forward(struct mbuf *m, int srcrt, st
sockaddr_in_init(&u.dst4, &ip->ip_dst, 0);
- if ((rt = rtcache_lookup(&ipforward_rt, &u.dst)) == NULL) {
+ ro = percpu_getref(ipforward_rt_percpu);
+ if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
+ percpu_putref(ipforward_rt_percpu);
icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
SOFTNET_UNLOCK();
return;
@@ -1376,7 +1386,7 @@ ip_forward(struct mbuf *m, int srcrt, st
}
}
- error = ip_output(m, NULL, &ipforward_rt,
+ error = ip_output(m, NULL, ro,
(IP_FORWARDING | (ip_directedbcast ? IP_ALLOWBROADCAST : 0)),
NULL, NULL);
@@ -1398,17 +1408,19 @@ ip_forward(struct mbuf *m, int srcrt, st
if (mcopy) {
#ifdef GATEWAY
if (mcopy->m_flags & M_CANFASTFWD)
- ipflow_create(&ipforward_rt, mcopy);
+ ipflow_create(ro, mcopy);
#endif
m_freem(mcopy);
}
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
return;
redirect:
error:
if (mcopy == NULL) {
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
return;
}
@@ -1432,7 +1444,7 @@ error:
type = ICMP_UNREACH;
code = ICMP_UNREACH_NEEDFRAG;
- if ((rt = rtcache_validate(&ipforward_rt)) != NULL)
+ if ((rt = rtcache_validate(ro)) != NULL)
destmtu = rt->rt_ifp->if_mtu;
#ifdef IPSEC
if (ipsec_used)
@@ -1450,10 +1462,12 @@ error:
*/
if (mcopy)
m_freem(mcopy);
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
return;
}
icmp_error(mcopy, type, code, dest, destmtu);
+ percpu_putref(ipforward_rt_percpu);
SOFTNET_UNLOCK();
}
Index: src/sys/netinet6/ip6_forward.c
diff -u src/sys/netinet6/ip6_forward.c:1.80 src/sys/netinet6/ip6_forward.c:1.81
--- src/sys/netinet6/ip6_forward.c:1.80 Tue Jun 28 02:02:56 2016
+++ src/sys/netinet6/ip6_forward.c Wed Aug 31 09:14:47 2016
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $ */
+/* $NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $ */
/* $KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $ */
/*
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -48,6 +48,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_forward.
#include <sys/time.h>
#include <sys/kernel.h>
#include <sys/syslog.h>
+#include <sys/percpu.h>
#include <net/if.h>
#include <net/route.h>
@@ -72,7 +73,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_forward.
#include <net/net_osdep.h>
-struct route ip6_forward_rt;
+extern percpu_t *ip6_forward_rt_percpu;
extern pfil_head_t *inet6_pfil_hook; /* XXX */
@@ -135,6 +136,7 @@ ip6_forward(struct mbuf *m, int srcrt)
struct in6_addr src_in6, dst_in6;
struct ifnet *rcvif = NULL;
struct psref psref;
+ struct route *ro;
#ifdef IPSEC
int needipsec = 0;
struct secpolicy *sp = NULL;
@@ -203,6 +205,7 @@ ip6_forward(struct mbuf *m, int srcrt)
}
#endif /* IPSEC */
+ ro = percpu_getref(ip6_forward_rt_percpu);
if (srcrt) {
union {
struct sockaddr dst;
@@ -210,17 +213,18 @@ ip6_forward(struct mbuf *m, int srcrt)
} u;
sockaddr_in6_init(&u.dst6, &ip6->ip6_dst, 0, 0, 0);
- if ((rt = rtcache_lookup(&ip6_forward_rt, &u.dst)) == NULL) {
+ if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
IP6_STATINC(IP6_STAT_NOROUTE);
/* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_noroute) */
if (mcopy) {
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOROUTE, 0);
}
+ percpu_putref(ip6_forward_rt_percpu);
goto drop;
}
- } else if ((rt = rtcache_validate(&ip6_forward_rt)) == NULL &&
- (rt = rtcache_update(&ip6_forward_rt, 1)) == NULL) {
+ } else if ((rt = rtcache_validate(ro)) == NULL &&
+ (rt = rtcache_update(ro, 1)) == NULL) {
/*
* rtcache_getdst(ip6_forward_rt)->sin6_addr was equal to
* ip6->ip6_dst
@@ -231,9 +235,11 @@ ip6_forward(struct mbuf *m, int srcrt)
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOROUTE, 0);
}
+ percpu_putref(ip6_forward_rt_percpu);
goto drop;
}
- dst = satocsin6(rtcache_getdst(&ip6_forward_rt));
+ dst = satocsin6(rtcache_getdst(ro));
+ percpu_putref(ip6_forward_rt_percpu);
/*
* Source scope check: if a packet can't be delivered to its
@@ -318,10 +324,11 @@ ip6_forward(struct mbuf *m, int srcrt)
*/
if (rt->rt_ifp == rcvif && !srcrt && ip6_sendredirects &&
(rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
+ ro = percpu_getref(ip6_forward_rt_percpu);
if ((rt->rt_ifp->if_flags & IFF_POINTOPOINT) &&
- nd6_is_addr_neighbor(
- satocsin6(rtcache_getdst(&ip6_forward_rt)),
- rt->rt_ifp)) {
+ nd6_is_addr_neighbor(satocsin6(rtcache_getdst(ro)),
+ rt->rt_ifp)) {
+ percpu_putref(ip6_forward_rt_percpu);
/*
* If the incoming interface is equal to the outgoing
* one, the link attached to the interface is
@@ -341,6 +348,7 @@ ip6_forward(struct mbuf *m, int srcrt)
ICMP6_DST_UNREACH_ADDR, 0);
goto drop;
}
+ percpu_putref(ip6_forward_rt_percpu);
type = ND_REDIRECT;
}
@@ -409,8 +417,10 @@ ip6_forward(struct mbuf *m, int srcrt)
IP6_STATINC(IP6_STAT_REDIRECTSENT);
else {
#ifdef GATEWAY
+ ro = percpu_getref(ip6_forward_rt_percpu);
if (m->m_flags & M_CANFASTFWD)
- ip6flow_create(&ip6_forward_rt, m);
+ ip6flow_create(ro, m);
+ percpu_putref(ip6_forward_rt_percpu);
#endif
if (mcopy)
goto freecopy;
Index: src/sys/netinet6/ip6_input.c
diff -u src/sys/netinet6/ip6_input.c:1.166 src/sys/netinet6/ip6_input.c:1.167
--- src/sys/netinet6/ip6_input.c:1.166 Tue Aug 2 04:50:16 2016
+++ src/sys/netinet6/ip6_input.c Wed Aug 31 09:14:47 2016
@@ -1,4 +1,4 @@
-/* $NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $ */
+/* $NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $ */
/* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */
/*
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_gateway.h"
@@ -87,6 +87,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_input.c,
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/cprng.h>
+#include <sys/percpu.h>
#include <net/if.h>
#include <net/if_types.h>
@@ -143,6 +144,8 @@ pfil_head_t *inet6_pfil_hook;
percpu_t *ip6stat_percpu;
+percpu_t *ip6_forward_rt_percpu __cacheline_aligned;
+
static void ip6_init2(void);
static void ip6intr(void *);
static struct m_tag *ip6_setdstifaddr(struct mbuf *, const struct in6_ifaddr *);
@@ -194,6 +197,10 @@ ip6_init(void)
KASSERT(inet6_pfil_hook != NULL);
ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS);
+
+ ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route));
+ if (ip6_forward_rt_percpu == NULL)
+ panic("failed to alllocate ip6_forward_rt_percpu");
}
static void
@@ -239,8 +246,6 @@ ip6intr(void *arg __unused)
mutex_exit(softnet_lock);
}
-extern struct route ip6_forward_rt;
-
void
ip6_input(struct mbuf *m, struct ifnet *rcvif)
{
@@ -256,6 +261,7 @@ ip6_input(struct mbuf *m, struct ifnet *
struct sockaddr dst;
struct sockaddr_in6 dst6;
} u;
+ struct route *ro;
/*
* make sure we don't have onion peering information into m_tag.
@@ -461,7 +467,9 @@ ip6_input(struct mbuf *m, struct ifnet *
/*
* Unicast check
*/
- rt = rtcache_lookup2(&ip6_forward_rt, &u.dst, 1, &hit);
+ ro = percpu_getref(ip6_forward_rt_percpu);
+ rt = rtcache_lookup2(ro, &u.dst, 1, &hit);
+ percpu_putref(ip6_forward_rt_percpu);
if (hit)
IP6_STATINC(IP6_STAT_FORWARD_CACHEHIT);
else
