Module Name:    src
Committed By:   ozaki-r
Date:           Thu Dec  8 05:16:34 UTC 2016

Modified Files:
        src/sys/dist/pf/net: pf.c
        src/sys/external/bsd/ipf/netinet: ip_fil_netbsd.c
        src/sys/net: if_stf.c route.h
        src/sys/netatalk: ddp_input.c ddp_output.c ddp_usrreq.c
        src/sys/netinet: in.c in_gif.c in_pcb.c in_pcb.h ip_etherip.c ip_flow.c
            ip_input.c ip_output.c ip_var.h sctp_output.c sctp_pcb.c
            sctp_timer.c tcp_input.c tcp_output.c tcp_subr.c
        src/sys/netinet6: frag6.c in6_gif.c in6_pcb.c in6_pcb.h in6_src.c
            ip6_etherip.c ip6_flow.c ip6_forward.c ip6_input.c ip6_output.c
            ip6_var.h
        src/sys/netipsec: ipsec.c

Log Message:
Add rtcache_unref to release points of rtentry stemming from rtcache

In the MP-safe world, a rtentry stemming from a rtcache can be freed at any
points. So we need to protect rtentries somehow say by reference couting or
passive references. Regardless of the method, we need to call some release
function of a rtentry after using it.

The change adds a new function rtcache_unref to release a rtentry. At this
point, this function does nothing because for now we don't add a reference
to a rtentry when we get one from a rtcache. We will add something useful
in a further commit.

This change is a part of changes for MP-safe routing table. It is separated
to avoid one big change that makes difficult to debug by bisecting.


To generate a diff of this commit:
cvs rdiff -u -r1.74 -r1.75 src/sys/dist/pf/net/pf.c
cvs rdiff -u -r1.18 -r1.19 src/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c
cvs rdiff -u -r1.99 -r1.100 src/sys/net/if_stf.c
cvs rdiff -u -r1.107 -r1.108 src/sys/net/route.h
cvs rdiff -u -r1.28 -r1.29 src/sys/netatalk/ddp_input.c
cvs rdiff -u -r1.19 -r1.20 src/sys/netatalk/ddp_output.c
cvs rdiff -u -r1.69 -r1.70 src/sys/netatalk/ddp_usrreq.c
cvs rdiff -u -r1.189 -r1.190 src/sys/netinet/in.c
cvs rdiff -u -r1.83 -r1.84 src/sys/netinet/in_gif.c
cvs rdiff -u -r1.170 -r1.171 src/sys/netinet/in_pcb.c
cvs rdiff -u -r1.60 -r1.61 src/sys/netinet/in_pcb.h
cvs rdiff -u -r1.17 -r1.18 src/sys/netinet/ip_etherip.c
cvs rdiff -u -r1.77 -r1.78 src/sys/netinet/ip_flow.c
cvs rdiff -u -r1.344 -r1.345 src/sys/netinet/ip_input.c
cvs rdiff -u -r1.263 -r1.264 src/sys/netinet/ip_output.c
cvs rdiff -u -r1.115 -r1.116 src/sys/netinet/ip_var.h
cvs rdiff -u -r1.7 -r1.8 src/sys/netinet/sctp_output.c \
    src/sys/netinet/sctp_pcb.c
cvs rdiff -u -r1.2 -r1.3 src/sys/netinet/sctp_timer.c
cvs rdiff -u -r1.349 -r1.350 src/sys/netinet/tcp_input.c
cvs rdiff -u -r1.186 -r1.187 src/sys/netinet/tcp_output.c
cvs rdiff -u -r1.267 -r1.268 src/sys/netinet/tcp_subr.c
cvs rdiff -u -r1.57 -r1.58 src/sys/netinet6/frag6.c
cvs rdiff -u -r1.79 -r1.80 src/sys/netinet6/in6_gif.c
cvs rdiff -u -r1.152 -r1.153 src/sys/netinet6/in6_pcb.c
cvs rdiff -u -r1.46 -r1.47 src/sys/netinet6/in6_pcb.h
cvs rdiff -u -r1.75 -r1.76 src/sys/netinet6/in6_src.c
cvs rdiff -u -r1.18 -r1.19 src/sys/netinet6/ip6_etherip.c
cvs rdiff -u -r1.32 -r1.33 src/sys/netinet6/ip6_flow.c
cvs rdiff -u -r1.81 -r1.82 src/sys/netinet6/ip6_forward.c
cvs rdiff -u -r1.170 -r1.171 src/sys/netinet6/ip6_input.c
cvs rdiff -u -r1.178 -r1.179 src/sys/netinet6/ip6_output.c
cvs rdiff -u -r1.70 -r1.71 src/sys/netinet6/ip6_var.h
cvs rdiff -u -r1.66 -r1.67 src/sys/netipsec/ipsec.c

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: src/sys/dist/pf/net/pf.c
diff -u src/sys/dist/pf/net/pf.c:1.74 src/sys/dist/pf/net/pf.c:1.75
--- src/sys/dist/pf/net/pf.c:1.74	Mon Jun 20 06:46:37 2016
+++ src/sys/dist/pf/net/pf.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: pf.c,v 1.74 2016/06/20 06:46:37 knakahara Exp $	*/
+/*	$NetBSD: pf.c,v 1.75 2016/12/08 05:16:33 ozaki-r Exp $	*/
 /*	$OpenBSD: pf.c,v 1.552.2.1 2007/11/27 16:37:57 henning Exp $ */
 
 /*
@@ -37,7 +37,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pf.c,v 1.74 2016/06/20 06:46:37 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pf.c,v 1.75 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #include "pflog.h"
 
@@ -2980,6 +2980,7 @@ pf_calc_mss(struct pf_addr *addr, sa_fam
 	if ((rt = rtcache_init_noclone(rop)) != NULL) {
 		mss = rt->rt_ifp->if_mtu - hlen - sizeof(struct tcphdr);
 		mss = max(tcp_mssdflt, mss);
+		rtcache_unref(rt, rop);
 	}
 	rtcache_free(rop);
 #endif
@@ -5068,6 +5069,7 @@ pf_routable(struct pf_addr *addr, sa_fam
 	} u;
 	struct route		 ro;
 	int			 ret = 1;
+	struct rtentry		*rt;
 
 	bzero(&ro, sizeof(ro));
 	switch (af) {
@@ -5084,7 +5086,10 @@ pf_routable(struct pf_addr *addr, sa_fam
 	}
 	rtcache_setdst(&ro, &u.dst);
 
-	ret = rtcache_init(&ro) != NULL ? 1 : 0;
+	rt = rtcache_init(&ro);
+	ret = rt != NULL ? 1 : 0;
+	if (rt != NULL)
+		rtcache_unref(rt, &ro);
 	rtcache_free(&ro);
 
 	return (ret);
@@ -5300,6 +5305,7 @@ pf_route(struct mbuf **m, struct pf_rule
 
 		if (rt->rt_flags & RTF_GATEWAY)
 			dst = rt->rt_gateway;
+		rtcache_unref(rt, ro); /* FIXME dst is NOMPSAFE */
 	} else {
 		if (TAILQ_EMPTY(&r->rpool.list)) {
 			DPFPRINTF(PF_DEBUG_URGENT,

Index: src/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c
diff -u src/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c:1.18 src/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c:1.19
--- src/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c:1.18	Mon Jul 18 21:07:30 2016
+++ src/sys/external/bsd/ipf/netinet/ip_fil_netbsd.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip_fil_netbsd.c,v 1.18 2016/07/18 21:07:30 pgoyette Exp $	*/
+/*	$NetBSD: ip_fil_netbsd.c,v 1.19 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 2012 by Darren Reed.
@@ -8,7 +8,7 @@
 #if !defined(lint)
 #if defined(__NetBSD__)
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_fil_netbsd.c,v 1.18 2016/07/18 21:07:30 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_fil_netbsd.c,v 1.19 2016/12/08 05:16:33 ozaki-r Exp $");
 #else
 static const char sccsid[] = "@(#)ip_fil.c	2.41 6/5/96 (C) 1993-2000 Darren Reed";
 static const char rcsid[] = "@(#)Id: ip_fil_netbsd.c,v 1.1.1.2 2012/07/22 13:45:17 darrenr Exp";
@@ -1330,6 +1330,7 @@ done:
 		softc->ipf_frouteok[1]++;
 
 # if __NetBSD_Version__ >= 499001100
+	rtcache_unref(rt, ro);
 	rtcache_free(ro);
 # else
 	if (rt) {
@@ -1467,6 +1468,7 @@ ipf_fastroute6(struct mbuf *m0, struct m
 	}
 bad:
 # if __NetBSD_Version__ >= 499001100
+	rtcache_unref(rt, ro);
 	rtcache_free(ro);
 # else
 	if (ro->ro_rt != NULL) {
@@ -1501,6 +1503,7 @@ ipf_verifysrc(fr_info_t *fin)
 		rc = 0;
 	else
 		rc = (fin->fin_ifp == rt->rt_ifp);
+	rtcache_unref(rt, &iproute);
 	rtcache_free(&iproute);
 #else
 	dst = (struct sockaddr_in *)&iproute.ro_dst;

Index: src/sys/net/if_stf.c
diff -u src/sys/net/if_stf.c:1.99 src/sys/net/if_stf.c:1.100
--- src/sys/net/if_stf.c:1.99	Thu Aug 18 11:38:58 2016
+++ src/sys/net/if_stf.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: if_stf.c,v 1.99 2016/08/18 11:38:58 knakahara Exp $	*/
+/*	$NetBSD: if_stf.c,v 1.100 2016/12/08 05:16:33 ozaki-r Exp $	*/
 /*	$KAME: if_stf.c,v 1.62 2001/06/07 22:32:16 itojun Exp $ */
 
 /*
@@ -75,7 +75,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_stf.c,v 1.99 2016/08/18 11:38:58 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_stf.c,v 1.100 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -446,11 +446,13 @@ stf_output(struct ifnet *ifp, struct mbu
 
 	/* If the route constitutes infinite encapsulation, punt. */
 	if (rt->rt_ifp == ifp) {
+		rtcache_unref(rt, &sc->sc_ro);
 		rtcache_free(&sc->sc_ro);
 		m_freem(m);
 		ifp->if_oerrors++;
 		return ENETUNREACH;
 	}
+	rtcache_unref(rt, &sc->sc_ro);
 
 	ifp->if_opackets++;
 	ifp->if_obytes += m->m_pkthdr.len - sizeof(struct ip);

Index: src/sys/net/route.h
diff -u src/sys/net/route.h:1.107 src/sys/net/route.h:1.108
--- src/sys/net/route.h:1.107	Tue Nov 15 01:50:06 2016
+++ src/sys/net/route.h	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: route.h,v 1.107 2016/11/15 01:50:06 ozaki-r Exp $	*/
+/*	$NetBSD: route.h,v 1.108 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -480,6 +480,13 @@ rtcache_validate(const struct route *ro)
 
 }
 
+static inline void
+rtcache_unref(struct rtentry *rt, struct route *ro)
+{
+
+	/* Will do something useful in the future. */
+}
+
 /* rtsock */
 void	rt_ieee80211msg(struct ifnet *, int, void *, size_t);
 void	rt_ifannouncemsg(struct ifnet *, int);

Index: src/sys/netatalk/ddp_input.c
diff -u src/sys/netatalk/ddp_input.c:1.28 src/sys/netatalk/ddp_input.c:1.29
--- src/sys/netatalk/ddp_input.c:1.28	Mon Oct  3 11:06:06 2016
+++ src/sys/netatalk/ddp_input.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ddp_input.c,v 1.28 2016/10/03 11:06:06 ozaki-r Exp $	 */
+/*	$NetBSD: ddp_input.c,v 1.29 2016/12/08 05:16:33 ozaki-r Exp $	 */
 
 /*
  * Copyright (c) 1990,1994 Regents of The University of Michigan.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ddp_input.c,v 1.28 2016/10/03 11:06:06 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ddp_input.c,v 1.29 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -279,9 +279,11 @@ ddp_input(struct mbuf *m, struct ifnet *
 		}
 #endif
 		if (ddp_firewall && (rt == NULL || rt->rt_ifp != ifp)) {
+			rtcache_unref(rt, &forwro);
 			m_freem(m);
 			return;
 		}
+		rtcache_unref(rt, &forwro);
 		ddpe.deh_hops++;
 		ddpe.deh_bytes = htonl(ddpe.deh_bytes);
 		memcpy((void *) deh, (void *) & ddpe, sizeof(u_short));/*XXX*/

Index: src/sys/netatalk/ddp_output.c
diff -u src/sys/netatalk/ddp_output.c:1.19 src/sys/netatalk/ddp_output.c:1.20
--- src/sys/netatalk/ddp_output.c:1.19	Mon Jun 20 06:46:38 2016
+++ src/sys/netatalk/ddp_output.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ddp_output.c,v 1.19 2016/06/20 06:46:38 knakahara Exp $	 */
+/*	$NetBSD: ddp_output.c,v 1.20 2016/12/08 05:16:33 ozaki-r Exp $	 */
 
 /*
  * Copyright (c) 1990,1991 Regents of The University of Michigan.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ddp_output.c,v 1.19 2016/06/20 06:46:38 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ddp_output.c,v 1.20 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -122,6 +122,7 @@ ddp_route(struct mbuf *m, struct route *
 	struct ifnet   *ifp = NULL;
 	uint16_t        net;
 	uint8_t         loopback = 0;
+	int		error;
 
 	if ((rt = rtcache_validate(ro)) != NULL && (ifp = rt->rt_ifp) != NULL) {
 		const struct sockaddr_at *dst = satocsat(rtcache_getdst(ro));
@@ -153,7 +154,8 @@ ddp_route(struct mbuf *m, struct route *
 		printf("%s: no address found\n", __func__);
 #endif
 		m_freem(m);
-		return EINVAL;
+		error = EINVAL;
+		goto out;
 	}
 	/*
          * There are several places in the kernel where data is added to
@@ -163,8 +165,10 @@ ddp_route(struct mbuf *m, struct route *
          */
 	if (!(aa->aa_flags & AFA_PHASE2)) {
 		M_PREPEND(m, SZ_ELAPHDR, M_DONTWAIT);
-		if (m == NULL)
-			return ENOBUFS;
+		if (m == NULL) {
+			error = ENOBUFS;
+			goto out;
+		}
 
 		elh = mtod(m, struct elaphdr *);
 		elh->el_snode = satosat(&aa->aa_addr)->sat_addr.s_node;
@@ -203,5 +207,9 @@ ddp_route(struct mbuf *m, struct route *
 #endif
 		looutput(lo0ifp, copym, rtcache_getdst(ro), NULL);
 	}
-	return if_output_lock(ifp, ifp, m, (struct sockaddr *)&gate, NULL);
+
+	error = if_output_lock(ifp, ifp, m, (struct sockaddr *)&gate, NULL);
+out:
+	rtcache_unref(rt, ro);
+	return error;
 }

Index: src/sys/netatalk/ddp_usrreq.c
diff -u src/sys/netatalk/ddp_usrreq.c:1.69 src/sys/netatalk/ddp_usrreq.c:1.70
--- src/sys/netatalk/ddp_usrreq.c:1.69	Mon Oct  3 11:06:06 2016
+++ src/sys/netatalk/ddp_usrreq.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ddp_usrreq.c,v 1.69 2016/10/03 11:06:06 ozaki-r Exp $	 */
+/*	$NetBSD: ddp_usrreq.c,v 1.70 2016/12/08 05:16:33 ozaki-r Exp $	 */
 
 /*
  * Copyright (c) 1990,1991 Regents of The University of Michigan.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ddp_usrreq.c,v 1.69 2016/10/03 11:06:06 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ddp_usrreq.c,v 1.70 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #include "opt_mbuftrace.h"
 
@@ -226,6 +226,7 @@ at_pcbconnect(struct ddpcb *ddp, struct 
 		if (aa == NULL || (cdst->sat_addr.s_net !=
 		    (hintnet ? hintnet : sat->sat_addr.s_net) ||
 		    cdst->sat_addr.s_node != sat->sat_addr.s_node)) {
+			rtcache_unref(rt, ro);
 			rtcache_free(ro);
 			rt = NULL;
 		}
@@ -254,6 +255,7 @@ at_pcbconnect(struct ddpcb *ddp, struct 
 		}
 	} else
 		aa = NULL;
+	rtcache_unref(rt, ro);
 	if (aa == NULL)
 		return ENETUNREACH;
 	ddp->ddp_fsat = *sat;

Index: src/sys/netinet/in.c
diff -u src/sys/netinet/in.c:1.189 src/sys/netinet/in.c:1.190
--- src/sys/netinet/in.c:1.189	Tue Dec  6 07:01:47 2016
+++ src/sys/netinet/in.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in.c,v 1.189 2016/12/06 07:01:47 knakahara Exp $	*/
+/*	$NetBSD: in.c,v 1.190 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in.c,v 1.189 2016/12/06 07:01:47 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in.c,v 1.190 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #include "arp.h"
 
@@ -1790,7 +1790,7 @@ in_selectsrc(struct sockaddr_in *sin, st
 		}
 		if (ia == NULL) {
 			*errorp = EADDRNOTAVAIL;
-			return NULL;
+			goto out;
 		}
 	}
 	/*
@@ -1820,7 +1820,8 @@ in_selectsrc(struct sockaddr_in *sin, st
 				if (ia != NULL)
 					ia4_release(ia, psref);
 				*errorp = EADDRNOTAVAIL;
-				return NULL;
+				ia = NULL;
+				goto out;
 			}
 			pserialize_read_exit(s);
 		}
@@ -1830,7 +1831,7 @@ in_selectsrc(struct sockaddr_in *sin, st
 		                                      sintosa(sin)));
 		if (ia == NULL) {
 			*errorp = EADDRNOTAVAIL;
-			return NULL;
+			goto out;
 		}
 		/* FIXME NOMPSAFE */
 		ia4_acquire(ia, psref);
@@ -1839,6 +1840,8 @@ in_selectsrc(struct sockaddr_in *sin, st
 	else
 		printf("%s: missing ifa_getifa\n", __func__);
 #endif
+out:
+	rtcache_unref(rt, ro);
 	return ia;
 }
 

Index: src/sys/netinet/in_gif.c
diff -u src/sys/netinet/in_gif.c:1.83 src/sys/netinet/in_gif.c:1.84
--- src/sys/netinet/in_gif.c:1.83	Thu Aug 18 11:34:09 2016
+++ src/sys/netinet/in_gif.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in_gif.c,v 1.83 2016/08/18 11:34:09 knakahara Exp $	*/
+/*	$NetBSD: in_gif.c,v 1.84 2016/12/08 05:16:33 ozaki-r Exp $	*/
 /*	$KAME: in_gif.c,v 1.66 2001/07/29 04:46:09 itojun Exp $	*/
 
 /*
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in_gif.c,v 1.83 2016/08/18 11:34:09 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in_gif.c,v 1.84 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -177,10 +177,12 @@ in_gif_output(struct ifnet *ifp, int fam
 
 	/* If the route constitutes infinite encapsulation, punt. */
 	if (rt->rt_ifp == ifp) {
+		rtcache_unref(rt, &sc->gif_ro);
 		rtcache_free(&sc->gif_ro);
 		m_freem(m);
 		return ENETUNREACH;	/*XXX*/
 	}
+	rtcache_unref(rt, &sc->gif_ro);
 
 	error = ip_output(m, NULL, &sc->gif_ro, 0, NULL, NULL);
 	return (error);

Index: src/sys/netinet/in_pcb.c
diff -u src/sys/netinet/in_pcb.c:1.170 src/sys/netinet/in_pcb.c:1.171
--- src/sys/netinet/in_pcb.c:1.170	Thu Sep 29 12:19:47 2016
+++ src/sys/netinet/in_pcb.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in_pcb.c,v 1.170 2016/09/29 12:19:47 roy Exp $	*/
+/*	$NetBSD: in_pcb.c,v 1.171 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -93,7 +93,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in_pcb.c,v 1.170 2016/09/29 12:19:47 roy Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in_pcb.c,v 1.171 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -777,8 +777,11 @@ in_pcbpurgeif(struct inpcbtable *table, 
 		if (inp->inp_af != AF_INET)
 			continue;
 		if ((rt = rtcache_validate(&inp->inp_route)) != NULL &&
-		    rt->rt_ifp == ifp)
+		    rt->rt_ifp == ifp) {
+			rtcache_unref(rt, &inp->inp_route);
 			in_rtchange(inp, 0);
+		} else
+			rtcache_unref(rt, &inp->inp_route);
 	}
 }
 
@@ -805,10 +808,17 @@ in_losing(struct inpcb *inp)
 	info.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
 	info.rti_info[RTAX_NETMASK] = rt_mask(rt);
 	rt_missmsg(RTM_LOSING, &info, rt->rt_flags, 0);
-	if (rt->rt_flags & RTF_DYNAMIC)
-		(void) rtrequest(RTM_DELETE, rt_getkey(rt),
-			rt->rt_gateway, rt_mask(rt), rt->rt_flags,
-			NULL);
+	if (rt->rt_flags & RTF_DYNAMIC) {
+		int error;
+		struct rtentry *nrt;
+
+		error = rtrequest(RTM_DELETE, rt_getkey(rt),
+		    rt->rt_gateway, rt_mask(rt), rt->rt_flags, &nrt);
+		rtcache_unref(rt, &inp->inp_route);
+		if (error == 0)
+			rtfree(nrt);
+	} else
+		rtcache_unref(rt, &inp->inp_route);
 	/*
 	 * A new route can be allocated
 	 * the next time output is attempted.
@@ -1088,3 +1098,10 @@ in_pcbrtentry(struct inpcb *inp)
 	sockaddr_in_init(&u.dst4, &inp->inp_faddr, 0);
 	return rtcache_lookup(ro, &u.dst);
 }
+
+void
+in_pcbrtentry_unref(struct rtentry *rt, struct inpcb *inp)
+{
+
+	rtcache_unref(rt, &inp->inp_route);
+}

Index: src/sys/netinet/in_pcb.h
diff -u src/sys/netinet/in_pcb.h:1.60 src/sys/netinet/in_pcb.h:1.61
--- src/sys/netinet/in_pcb.h:1.60	Tue Apr 26 08:44:44 2016
+++ src/sys/netinet/in_pcb.h	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in_pcb.h,v 1.60 2016/04/26 08:44:44 ozaki-r Exp $	*/
+/*	$NetBSD: in_pcb.h,v 1.61 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -160,6 +160,7 @@ void	in_setpeeraddr(struct inpcb *, stru
 void	in_setsockaddr(struct inpcb *, struct sockaddr_in *);
 struct rtentry *
 	in_pcbrtentry(struct inpcb *);
+void	in_pcbrtentry_unref(struct rtentry *, struct inpcb *);
 #endif
 
 #endif /* !_NETINET_IN_PCB_H_ */

Index: src/sys/netinet/ip_etherip.c
diff -u src/sys/netinet/ip_etherip.c:1.17 src/sys/netinet/ip_etherip.c:1.18
--- src/sys/netinet/ip_etherip.c:1.17	Fri Jun 10 13:27:16 2016
+++ src/sys/netinet/ip_etherip.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*      $NetBSD: ip_etherip.c,v 1.17 2016/06/10 13:27:16 ozaki-r Exp $        */
+/*      $NetBSD: ip_etherip.c,v 1.18 2016/12/08 05:16:33 ozaki-r Exp $        */
 
 /*
  *  Copyright (c) 2006, Hans Rosenfeld <rosenf...@grumpf.hope-2000.org>
@@ -58,7 +58,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_etherip.c,v 1.17 2016/06/10 13:27:16 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_etherip.c,v 1.18 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -167,10 +167,12 @@ ip_etherip_output(struct ifnet *ifp, str
 
 	/* if it constitutes infinite encapsulation, punt. */
 	if (rt->rt_ifp == ifp) {
+		rtcache_unref(rt, &sc->sc_ro);
 		rtcache_free(&sc->sc_ro);
 		m_freem(m);
 		return ENETUNREACH;     /*XXX*/
 	}
+	rtcache_unref(rt, &sc->sc_ro);
 
 	error = ip_output(m, NULL, &sc->sc_ro, 0, NULL, NULL);
 

Index: src/sys/netinet/ip_flow.c
diff -u src/sys/netinet/ip_flow.c:1.77 src/sys/netinet/ip_flow.c:1.78
--- src/sys/netinet/ip_flow.c:1.77	Tue Oct 18 07:30:31 2016
+++ src/sys/netinet/ip_flow.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip_flow.c,v 1.77 2016/10/18 07:30:31 ozaki-r Exp $	*/
+/*	$NetBSD: ip_flow.c,v 1.78 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_flow.c,v 1.77 2016/10/18 07:30:31 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_flow.c,v 1.78 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_net_mpsafe.h"
@@ -208,7 +208,7 @@ ipflow_fastforward(struct mbuf *m)
 	struct ip *ip;
 	struct ip ip_store;
 	struct ipflow *ipf;
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 	const struct sockaddr *dst;
 	int error;
 	int iplen;
@@ -258,7 +258,7 @@ ipflow_fastforward(struct mbuf *m)
 		 M_CSUM_IPv4_BAD)) {
 	case M_CSUM_IPv4|M_CSUM_IPv4_BAD:
 		m_put_rcvif(ifp, &s);
-		goto out;
+		goto out_unref;
 
 	case M_CSUM_IPv4:
 		/* Checksum was okay. */
@@ -268,7 +268,7 @@ ipflow_fastforward(struct mbuf *m)
 		/* Must compute it ourselves. */
 		if (in_cksum(m, sizeof(struct ip)) != 0) {
 			m_put_rcvif(ifp, &s);
-			goto out;
+			goto out_unref;
 		}
 		break;
 	}
@@ -277,16 +277,16 @@ ipflow_fastforward(struct mbuf *m)
 	/*
 	 * Route and interface still up?
 	 */
-	if ((rt = rtcache_validate(&ipf->ipf_ro)) == NULL ||
-	    (rt->rt_ifp->if_flags & IFF_UP) == 0 ||
+	rt = rtcache_validate(&ipf->ipf_ro);
+	if (rt == NULL || (rt->rt_ifp->if_flags & IFF_UP) == 0 ||
 	    (rt->rt_flags & (RTF_BLACKHOLE | RTF_BROADCAST)) != 0)
-		goto out;
+		goto out_unref;
 
 	/*
 	 * Packet size OK?  TTL?
 	 */
 	if (m->m_pkthdr.len > rt->rt_ifp->if_mtu || ip->ip_ttl <= IPTTLDEC)
-		goto out;
+		goto out_unref;
 
 	/*
 	 * Clear any in-bound checksum flags for this packet.
@@ -359,7 +359,9 @@ ipflow_fastforward(struct mbuf *m)
 			ipf->ipf_errors++;
 	}
 	ret = 1;
- out:
+out_unref:
+	rtcache_unref(rt, &ipf->ipf_ro);
+out:
 	mutex_exit(&ipflow_lock);
 	return ret;
 }
@@ -370,8 +372,11 @@ ipflow_addstats(struct ipflow *ipf)
 	struct rtentry *rt;
 	uint64_t *ips;
 
-	if ((rt = rtcache_validate(&ipf->ipf_ro)) != NULL)
+	rt = rtcache_validate(&ipf->ipf_ro);
+	if (rt != NULL) {
 		rt->rt_use += ipf->ipf_uses;
+		rtcache_unref(rt, &ipf->ipf_ro);
+	}
 	
 	ips = IP_STAT_GETREF();
 	ips[IP_STAT_CANTFORWARD] += ipf->ipf_errors + ipf->ipf_dropped;
@@ -431,12 +436,15 @@ ipflow_reap(bool just_one)
 		struct ipflow *maybe_ipf = TAILQ_LAST(&ipflowlist, ipflowhead);
 
 		TAILQ_FOREACH(ipf, &ipflowlist, ipf_list) {
+			struct rtentry *rt;
 			/*
 			 * If this no longer points to a valid route
 			 * reclaim it.
 			 */
-			if (rtcache_validate(&ipf->ipf_ro) == NULL)
+			rt = rtcache_validate(&ipf->ipf_ro);
+			if (rt == NULL)
 				goto done;
+			rtcache_unref(rt, &ipf->ipf_ro);
 			/*
 			 * choose the one that's been least recently
 			 * used or has had the least uses in the
@@ -488,6 +496,7 @@ ipflow_slowtimo_work(struct work *wk, vo
 		} else {
 			ipf->ipf_last_uses = ipf->ipf_uses;
 			rt->rt_use += ipf->ipf_uses;
+			rtcache_unref(rt, &ipf->ipf_ro);
 			ips = IP_STAT_GETREF();
 			ips[IP_STAT_TOTAL] += ipf->ipf_uses;
 			ips[IP_STAT_FORWARD] += ipf->ipf_uses;
@@ -515,7 +524,7 @@ ipflow_slowtimo(void)
 }
 
 void
-ipflow_create(const struct route *ro, struct mbuf *m)
+ipflow_create(struct route *ro, struct mbuf *m)
 {
 	const struct ip *const ip = mtod(m, const struct ip *);
 	struct ipflow *ipf;

Index: src/sys/netinet/ip_input.c
diff -u src/sys/netinet/ip_input.c:1.344 src/sys/netinet/ip_input.c:1.345
--- src/sys/netinet/ip_input.c:1.344	Tue Oct 18 07:30:31 2016
+++ src/sys/netinet/ip_input.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip_input.c,v 1.344 2016/10/18 07:30:31 ozaki-r Exp $	*/
+/*	$NetBSD: ip_input.c,v 1.345 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.344 2016/10/18 07:30:31 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.345 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -1183,13 +1183,15 @@ ip_rtaddr(struct in_addr dst)
 
 	sockaddr_in_init(&u.dst4, &dst, 0);
 
-	SOFTNET_LOCK();
 	ro = percpu_getref(ipforward_rt_percpu);
 	rt = rtcache_lookup(ro, &u.dst);
-	percpu_putref(ipforward_rt_percpu);
-	SOFTNET_UNLOCK();
-	if (rt == NULL)
+	if (rt == NULL) {
+		percpu_putref(ipforward_rt_percpu);
 		return NULL;
+	}
+
+	rtcache_unref(rt, ro);
+	percpu_putref(ipforward_rt_percpu);
 
 	return ifatoia(rt->rt_ifa);
 }
@@ -1349,7 +1351,8 @@ ip_forward(struct mbuf *m, int srcrt, st
 	sockaddr_in_init(&u.dst4, &ip->ip_dst, 0);
 
 	ro = percpu_getref(ipforward_rt_percpu);
-	if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
+	rt = rtcache_lookup(ro, &u.dst);
+	if (rt == NULL) {
 		percpu_putref(ipforward_rt_percpu);
 		icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
 		return;
@@ -1393,6 +1396,7 @@ ip_forward(struct mbuf *m, int srcrt, st
 			code = ICMP_REDIRECT_HOST;
 		}
 	}
+	rtcache_unref(rt, ro);
 
 	error = ip_output(m, NULL, ro,
 	    (IP_FORWARDING | (ip_directedbcast ? IP_ALLOWBROADCAST : 0)),
@@ -1450,8 +1454,10 @@ error:
 		type = ICMP_UNREACH;
 		code = ICMP_UNREACH_NEEDFRAG;
 
-		if ((rt = rtcache_validate(ro)) != NULL)
+		if ((rt = rtcache_validate(ro)) != NULL) {
 			destmtu = rt->rt_ifp->if_mtu;
+			rtcache_unref(rt, ro);
+		}
 #ifdef IPSEC
 		if (ipsec_used)
 			(void)ipsec4_forward(mcopy, &destmtu);

Index: src/sys/netinet/ip_output.c
diff -u src/sys/netinet/ip_output.c:1.263 src/sys/netinet/ip_output.c:1.264
--- src/sys/netinet/ip_output.c:1.263	Tue Sep 20 14:30:13 2016
+++ src/sys/netinet/ip_output.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip_output.c,v 1.263 2016/09/20 14:30:13 roy Exp $	*/
+/*	$NetBSD: ip_output.c,v 1.264 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip_output.c,v 1.263 2016/09/20 14:30:13 roy Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip_output.c,v 1.264 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -765,6 +765,7 @@ sendit:
 	}
 done:
 	ia4_release(ia, &psref_ia);
+	rtcache_unref(rt, ro);
 	if (ro == &iproute) {
 		rtcache_free(&iproute);
 	}
@@ -1543,6 +1544,7 @@ ip_get_membership(const struct sockopt *
 		if (error != 0)
 			return error;
 		*ifp = (rt = rtcache_init(&ro)) != NULL ? rt->rt_ifp : NULL;
+		rtcache_unref(rt, &ro);
 		rtcache_free(&ro);
 	} else {
 		*ifp = ip_multicast_if(&mreq.imr_interface, NULL);

Index: src/sys/netinet/ip_var.h
diff -u src/sys/netinet/ip_var.h:1.115 src/sys/netinet/ip_var.h:1.116
--- src/sys/netinet/ip_var.h:1.115	Mon Aug  1 10:22:53 2016
+++ src/sys/netinet/ip_var.h	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip_var.h,v 1.115 2016/08/01 10:22:53 knakahara Exp $	*/
+/*	$NetBSD: ip_var.h,v 1.116 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -245,7 +245,7 @@ int	ip_if_output(struct ifnet * const, s
 /* IP Flow interface. */
 void	ipflow_init(void);
 void	ipflow_poolinit(void);
-void	ipflow_create(const struct route *, struct mbuf *);
+void	ipflow_create(struct route *, struct mbuf *);
 void	ipflow_slowtimo(void);
 int	ipflow_invalidate_all(int);
 

Index: src/sys/netinet/sctp_output.c
diff -u src/sys/netinet/sctp_output.c:1.7 src/sys/netinet/sctp_output.c:1.8
--- src/sys/netinet/sctp_output.c:1.7	Thu Jul  7 09:32:02 2016
+++ src/sys/netinet/sctp_output.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: sctp_output.c,v 1.7 2016/07/07 09:32:02 ozaki-r Exp $ */
+/*	$NetBSD: sctp_output.c,v 1.8 2016/12/08 05:16:33 ozaki-r Exp $ */
 /*	$KAME: sctp_output.c,v 1.48 2005/06/16 18:29:24 jinmei Exp $	*/
 
 /*
@@ -30,7 +30,7 @@
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sctp_output.c,v 1.7 2016/07/07 09:32:02 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sctp_output.c,v 1.8 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ipsec.h"
@@ -1083,6 +1083,7 @@ sctp_ipv4_source_address_selection(struc
 	const struct sockaddr_in *to;
 	struct rtentry *rt;
 	uint8_t ipv4_scope, loopscope;
+
 	/*
 	 * Rules:
 	 * - Find the route if needed, cache if I can.
@@ -1176,8 +1177,9 @@ sctp_ipv4_source_address_selection(struc
 		 * it is a negative list. Addresses being added
 		 * by asconf.
 		 */
-		return (sctp_choose_v4_boundall(inp, stcb, net, rt,
-		    ipv4_scope, loopscope, non_asoc_addr_ok));
+		ans = sctp_choose_v4_boundall(inp, stcb, net, rt,
+		    ipv4_scope, loopscope, non_asoc_addr_ok);
+		goto out;
         }
 	/*
  	 * Three possiblities here:
@@ -1199,15 +1201,19 @@ sctp_ipv4_source_address_selection(struc
 	 *    the v6 address selection.
 	 */
 	if (stcb) {
-		return (sctp_choose_v4_boundspecific_stcb(inp, stcb, net,
-		    rt, ipv4_scope, loopscope, non_asoc_addr_ok));
+		ans = sctp_choose_v4_boundspecific_stcb(inp, stcb, net,
+		    rt, ipv4_scope, loopscope, non_asoc_addr_ok);
+		goto out;
 	} else {
-		return (sctp_choose_v4_boundspecific_inp(inp, rt,
-		    ipv4_scope, loopscope));
+		ans = sctp_choose_v4_boundspecific_inp(inp, rt,
+		    ipv4_scope, loopscope);
+		goto out;
 	}
 	/* this should not be reached */
 	memset(&ans, 0, sizeof(ans));
-	return (ans);
+out:
+	rtcache_unref(rt, ro);
+	return ans;
 }
 
 
@@ -1980,6 +1986,7 @@ sctp_ipv6_source_address_selection(struc
 			/* we can't have a non-asoc address since we have no association */
 			rt_addr = sctp_choose_v6_boundspecific_inp(inp,  rt, loc_scope, loopscope);
 	}
+	rtcache_unref(rt, ro);
 	if (rt_addr == NULL) {
 		/* no suitable address? */
 		struct in6_addr in6;
@@ -2150,7 +2157,9 @@ sctp_lowlevel_chunk_output(struct sctp_i
 		if (net == NULL) {
 			ro = &iproute;
 			memset(&iproute, 0, sizeof(iproute));
-			rtcache_lookup(ro, to);
+			/* XXX */
+			rt = rtcache_lookup(ro, to);
+			rtcache_unref(rt, ro);
 		} else {
 			ro = (struct route *)&net->ro;
 		}
@@ -2164,9 +2173,11 @@ sctp_lowlevel_chunk_output(struct sctp_i
 				((struct sockaddr_in *)&net->_s_addr)->sin_addr = sctp_ipv4_source_address_selection(inp,
 				    stcb,
 				    ro, net, out_of_asoc_ok);
-				if (rtcache_validate(ro)) {
+				rt = rtcache_validate(ro);
+				if (rt != NULL) {
 					net->src_addr_selected = 1;
 				}
+				rtcache_unref(rt, ro);
 			}
 			ip->ip_src = ((struct sockaddr_in *)&net->_s_addr)->sin_addr;
 		} else {
@@ -2264,6 +2275,7 @@ sctp_lowlevel_chunk_output(struct sctp_i
 				net->src_addr_selected = 0;
 			}
 		}
+		rtcache_unref(rt, ro);
 		return (ret);
 	}
 #ifdef INET6
@@ -2312,7 +2324,9 @@ sctp_lowlevel_chunk_output(struct sctp_i
 		if (net == NULL) {
 			memset(&ip6route, 0, sizeof(ip6route));
 			ro = (struct route *)&ip6route;
-			rtcache_lookup(ro, (struct sockaddr *) sin6);
+			/* XXX */
+			rt = rtcache_lookup(ro, (struct sockaddr *) sin6);
+			rtcache_unref(rt, ro);
 		} else {
 			ro = (struct route *)&net->ro;
 		}
@@ -2429,6 +2443,8 @@ sctp_lowlevel_chunk_output(struct sctp_i
 			prev_scope = sin6->sin6_scope_id;
 			prev_port = sin6->sin6_port;
 		}
+		/* XXX NOMPSAFE need to hold ifp here */
+		rtcache_unref(rt, ro);
 		ret = ip6_output(m, ((struct in6pcb *)inp)->in6p_outputopts,
 				 ro,
 				 o_flgs,
@@ -2460,6 +2476,7 @@ sctp_lowlevel_chunk_output(struct sctp_i
 							    &stcb->asoc,
 							    rt->rt_rmx.rmx_mtu);
 				}
+				rtcache_unref(rt, ro);
 			} else if (ifp) {
 				if (ND_IFINFO(ifp)->linkmtu &&
 				    (stcb->asoc.smallest_mtu > ND_IFINFO(ifp)->linkmtu)) {
@@ -3252,6 +3269,7 @@ sctp_send_initiate_ack(struct sctp_inpcb
 	int cnt_inits_to=0;
 	uint16_t his_limit, i_want;
 	int abort_flag, padval, sz_of;
+	struct rtentry *rt;
 
 	if (stcb) {
 		asoc = &stcb->asoc;
@@ -3402,7 +3420,9 @@ sctp_send_initiate_ack(struct sctp_inpcb
 			memset(&iproute, 0, sizeof(iproute));
 			ro = &iproute;
 
-			rtcache_lookup(ro, (struct sockaddr *) sin);
+			/* XXX */
+			rt = rtcache_lookup(ro, (struct sockaddr *) sin);
+			rtcache_unref(rt, ro);
 			addr = sctp_ipv4_source_address_selection(inp, NULL,
 			    ro, NULL, 0);
 			stc.laddress[0] = addr.s_addr;
@@ -3488,7 +3508,9 @@ sctp_send_initiate_ack(struct sctp_inpcb
 			/* local from address */
 			memset(&iproute6, 0, sizeof(iproute6));
 			ro = (struct route *)&iproute6;
-			rtcache_lookup(ro, (struct sockaddr *) sin6);
+			/* XXX */
+			rt = rtcache_lookup(ro, (struct sockaddr *) sin6);
+			rtcache_unref(rt, ro);
 			addr = sctp_ipv6_source_address_selection(inp, NULL,
 			    ro, NULL, 0);
 			memcpy(&stc.laddress, &addr, sizeof(struct in6_addr));
@@ -5401,6 +5423,7 @@ sctp_med_chunk_output(struct sctp_inpcb 
 		return (0);
 	}
 	TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
+		struct rtentry *rt;
 		/* how much can we send? */
 		if (net->ref_count < 2) {
 			/* Ref-count of 1 so we cannot have data or control
@@ -5413,7 +5436,8 @@ sctp_med_chunk_output(struct sctp_inpcb 
 		no_fragmentflg = 1;
 		one_chunk = 0;
 
-		if (rtcache_validate(&net->ro)) {
+		rt = rtcache_validate(&net->ro);
+		if (rt != NULL) {
 			/* if we have a route and an ifp
 			 * check to see if we have room to
 			 * send to this guy
@@ -5425,8 +5449,10 @@ sctp_med_chunk_output(struct sctp_inpcb 
 #ifdef SCTP_LOG_MAXBURST
 				sctp_log_maxburst(net, ifp->if_snd.ifq_len, ifp->if_snd.ifq_maxlen, SCTP_MAX_IFP_APPLIED);
   #endif
+				rtcache_unref(rt, &net->ro);
 				continue;
 			}
+			rtcache_unref(rt, &net->ro);
 		}
 		if (((struct sockaddr *)&net->ro.ro_sa)->sa_family == AF_INET) {
 			mtu = net->mtu - (sizeof(struct ip) + sizeof(struct sctphdr));
Index: src/sys/netinet/sctp_pcb.c
diff -u src/sys/netinet/sctp_pcb.c:1.7 src/sys/netinet/sctp_pcb.c:1.8
--- src/sys/netinet/sctp_pcb.c:1.7	Thu Jul  7 09:32:02 2016
+++ src/sys/netinet/sctp_pcb.c	Thu Dec  8 05:16:33 2016
@@ -1,5 +1,5 @@
 /* $KAME: sctp_pcb.c,v 1.39 2005/06/16 18:29:25 jinmei Exp $ */
-/* $NetBSD: sctp_pcb.c,v 1.7 2016/07/07 09:32:02 ozaki-r Exp $ */
+/* $NetBSD: sctp_pcb.c,v 1.8 2016/12/08 05:16:33 ozaki-r Exp $ */
 
 /*
  * Copyright (c) 2001, 2002, 2003, 2004 Cisco Systems, Inc.
@@ -33,7 +33,7 @@
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sctp_pcb.c,v 1.7 2016/07/07 09:32:02 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sctp_pcb.c,v 1.8 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -2061,6 +2061,7 @@ sctp_inpcb_free(struct sctp_inpcb *inp, 
 	struct socket *so;
 	struct sctp_socket_q_list *sq;
 	int s, cnt;
+	struct rtentry *rt;
 
 	s = splsoftnet();
 	SCTP_ASOC_CREATE_LOCK(inp);
@@ -2173,7 +2174,9 @@ sctp_inpcb_free(struct sctp_inpcb *inp, 
 #endif
 	inp->sctp_flags |= SCTP_PCB_FLAGS_SOCKET_ALLGONE;
 
-	rtcache_validate(&ip_pcb->inp_route);
+	/* XXX */
+	rt = rtcache_validate(&ip_pcb->inp_route);
+	rtcache_unref(rt, &ip_pcb->inp_route);
 
 	callout_stop(&inp->sctp_ep.signature_change.timer);
 	callout_destroy(&inp->sctp_ep.signature_change.timer);
@@ -2608,6 +2611,7 @@ sctp_add_remote_addr(struct sctp_tcb *st
 		 */
 		TAILQ_INSERT_HEAD(&stcb->asoc.nets, net, sctp_next);
 	} else if (rt->rt_ifp != netfirst_rt->rt_ifp) {
+		rtcache_unref(netfirst_rt, &netfirst->ro);
 		/*
 		 * This one has a different interface than the one at the
 		 * top of the list. Place it ahead.
@@ -2635,13 +2639,16 @@ sctp_add_remote_addr(struct sctp_tcb *st
 				TAILQ_INSERT_BEFORE(netfirst, net, sctp_next);
 				break;
 			} else if (netlook_rt->rt_ifp != rt->rt_ifp) {
+				rtcache_unref(netlook_rt, &netlook->ro);
 				TAILQ_INSERT_AFTER(&stcb->asoc.nets, netlook,
 				    net, sctp_next);
 				break;
 			}
+			rtcache_unref(netlook_rt, &netlook->ro);
 			/* Shift forward */
 			netfirst = netlook;
 		} while (netlook != NULL);
+		rtcache_unref(netfirst_rt, &netfirst->ro);
 	}
 	/* got to have a primary set */
 	if (stcb->asoc.primary_destination == 0) {

Index: src/sys/netinet/sctp_timer.c
diff -u src/sys/netinet/sctp_timer.c:1.2 src/sys/netinet/sctp_timer.c:1.3
--- src/sys/netinet/sctp_timer.c:1.2	Mon Apr 25 21:21:02 2016
+++ src/sys/netinet/sctp_timer.c	Thu Dec  8 05:16:33 2016
@@ -1,5 +1,5 @@
 /*	$KAME: sctp_timer.c,v 1.30 2005/06/16 18:29:25 jinmei Exp $	*/
-/*	$NetBSD: sctp_timer.c,v 1.2 2016/04/25 21:21:02 rjs Exp $	*/
+/*	$NetBSD: sctp_timer.c,v 1.3 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 2002, 2003, 2004 Cisco Systems Inc,
@@ -30,7 +30,7 @@
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sctp_timer.c,v 1.2 2016/04/25 21:21:02 rjs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sctp_timer.c,v 1.3 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -247,8 +247,10 @@ sctp_find_alternate_net(struct sctp_tcb 
 			(!(alt->dest_state & SCTP_ADDR_UNCONFIRMED))
 			) {
 			/* Found a reachable address */
+			rtcache_unref(rt, &alt->ro);
 			break;
 		}
+		rtcache_unref(rt, &alt->ro);
 		mnet = alt;
 	} while (alt != NULL);
 
@@ -1346,6 +1348,7 @@ void sctp_pathmtu_timer(struct sctp_inpc
 				net->mtu = next_mtu;
 			}
 		}
+		rtcache_unref(rt, &net->ro);
 	}
 	/* restart the timer */
 	sctp_timer_start(SCTP_TIMER_TYPE_PATHMTURAISE, inp, stcb, net);

Index: src/sys/netinet/tcp_input.c
diff -u src/sys/netinet/tcp_input.c:1.349 src/sys/netinet/tcp_input.c:1.350
--- src/sys/netinet/tcp_input.c:1.349	Tue Nov 15 22:23:09 2016
+++ src/sys/netinet/tcp_input.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: tcp_input.c,v 1.349 2016/11/15 22:23:09 mrg Exp $	*/
+/*	$NetBSD: tcp_input.c,v 1.350 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -148,7 +148,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tcp_input.c,v 1.349 2016/11/15 22:23:09 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tcp_input.c,v 1.350 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -262,11 +262,12 @@ static struct timeval tcp_ackdrop_ppslim
 static inline void
 nd6_hint(struct tcpcb *tp)
 {
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 
 	if (tp != NULL && tp->t_in6pcb != NULL && tp->t_family == AF_INET6 &&
 	    (rt = rtcache_validate(&tp->t_in6pcb->in6p_route)) != NULL)
 		nd6_nud_hint(rt);
+	rtcache_unref(rt, &tp->t_in6pcb->in6p_route);
 }
 #else
 static inline void
@@ -4528,7 +4529,7 @@ int
 syn_cache_respond(struct syn_cache *sc, struct mbuf *m)
 {
 #ifdef INET6
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 #endif
 	struct route *ro;
 	u_int8_t *optp;
@@ -4809,6 +4810,7 @@ syn_cache_respond(struct syn_cache *sc, 
 	case AF_INET6:
 		ip6->ip6_hlim = in6_selecthlim(NULL,
 		    (rt = rtcache_validate(ro)) != NULL ? rt->rt_ifp : NULL);
+		rtcache_unref(rt, ro);
 
 		error = ip6_output(m, NULL /*XXX*/, ro, 0, NULL, so, NULL);
 		break;

Index: src/sys/netinet/tcp_output.c
diff -u src/sys/netinet/tcp_output.c:1.186 src/sys/netinet/tcp_output.c:1.187
--- src/sys/netinet/tcp_output.c:1.186	Fri Jun 10 13:27:16 2016
+++ src/sys/netinet/tcp_output.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: tcp_output.c,v 1.186 2016/06/10 13:27:16 ozaki-r Exp $	*/
+/*	$NetBSD: tcp_output.c,v 1.187 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -135,7 +135,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tcp_output.c,v 1.186 2016/06/10 13:27:16 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tcp_output.c,v 1.187 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -338,6 +338,14 @@ tcp_segsize(struct tcpcb *tp, int *txseg
 		}
 	}
 #endif
+#ifdef INET
+	if (inp)
+		in_pcbrtentry_unref(rt, inp);
+#endif
+#ifdef INET6
+	if (in6p)
+		in6_pcbrtentry_unref(rt, in6p);
+#endif
  out:
 	/*
 	 * Now we must make room for whatever extra TCP/IP options are in
@@ -547,7 +555,7 @@ tcp_build_datapkt(struct tcpcb *tp, stru
 int
 tcp_output(struct tcpcb *tp)
 {
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 	struct socket *so;
 	struct route *ro;
 	long len, win;
@@ -638,6 +646,10 @@ tcp_output(struct tcpcb *tp)
 #endif
 	    (rt = rtcache_validate(&tp->t_inpcb->inp_route)) != NULL &&
 	    (rt->rt_ifp->if_capenable & IFCAP_TSOv4) != 0;
+	if (rt != NULL) {
+		rtcache_unref(rt, &tp->t_inpcb->inp_route);
+		rt = NULL;
+	}
 #endif /* defined(INET) */
 #if defined(INET6)
 	has_tso6 = tp->t_in6pcb != NULL &&
@@ -647,6 +659,8 @@ tcp_output(struct tcpcb *tp)
 #endif
 	    (rt = rtcache_validate(&tp->t_in6pcb->in6p_route)) != NULL &&
 	    (rt->rt_ifp->if_capenable & IFCAP_TSOv6) != 0;
+	if (rt != NULL)
+		rtcache_unref(rt, &tp->t_in6pcb->in6p_route);
 #endif /* defined(INET6) */
 	has_tso = (has_tso4 || has_tso6) && !alwaysfrag;
 
@@ -1134,6 +1148,14 @@ send:
 		tp->snd_nxt = tp->iss;
 		tp->t_ourmss = tcp_mss_to_advertise(synrt != NULL ?
 						    synrt->rt_ifp : NULL, af);
+#ifdef INET
+		if (tp->t_inpcb)
+			in_pcbrtentry_unref(synrt, tp->t_inpcb);
+#endif
+#ifdef INET6
+		if (tp->t_in6pcb)
+			in6_pcbrtentry_unref(synrt, tp->t_in6pcb);
+#endif
 		if ((tp->t_flags & TF_NOOPT) == 0 && OPT_FITS(4)) {
 			opt[0] = TCPOPT_MAXSEG;
 			opt[1] = 4;

Index: src/sys/netinet/tcp_subr.c
diff -u src/sys/netinet/tcp_subr.c:1.267 src/sys/netinet/tcp_subr.c:1.268
--- src/sys/netinet/tcp_subr.c:1.267	Wed Nov  9 03:33:30 2016
+++ src/sys/netinet/tcp_subr.c	Thu Dec  8 05:16:33 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: tcp_subr.c,v 1.267 2016/11/09 03:33:30 ozaki-r Exp $	*/
+/*	$NetBSD: tcp_subr.c,v 1.268 2016/12/08 05:16:33 ozaki-r Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -91,7 +91,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: tcp_subr.c,v 1.267 2016/11/09 03:33:30 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: tcp_subr.c,v 1.268 2016/12/08 05:16:33 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -1156,7 +1156,7 @@ tcp_close(struct tcpcb *tp)
 #endif
 	struct socket *so;
 #ifdef RTV_RTT
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 #endif
 	struct route *ro;
 	int j;
@@ -1245,6 +1245,7 @@ tcp_close(struct tcpcb *tp)
 				rt->rt_rmx.rmx_ssthresh = i;
 		}
 	}
+	rtcache_unref(rt, ro);
 #endif /* RTV_RTT */
 	/* free the reassembly queue, if any */
 	TCP_REASS_LOCK(tp);
@@ -1774,6 +1775,7 @@ tcp_mtudisc(struct inpcb *inp, int errno
 		 * If this was not a host route, remove and realloc.
 		 */
 		if ((rt->rt_flags & RTF_HOST) == 0) {
+			in_pcbrtentry_unref(rt, inp);
 			in_rtchange(inp, errno);
 			if ((rt = in_pcbrtentry(inp)) == NULL)
 				return;
@@ -1791,6 +1793,7 @@ tcp_mtudisc(struct inpcb *inp, int errno
 			tp->snd_cwnd =
 			    TCP_INITIAL_WINDOW(tcp_init_win,
 			    rt->rt_rmx.rmx_mtu);
+		in_pcbrtentry_unref(rt, inp);
 	}
 
 	/*
@@ -1833,6 +1836,7 @@ tcp6_mtudisc(struct in6pcb *in6p, int er
 		 * If this was not a host route, remove and realloc.
 		 */
 		if ((rt->rt_flags & RTF_HOST) == 0) {
+			in6_pcbrtentry_unref(rt, in6p);
 			in6_rtchange(in6p, errno);
 			rt = in6_pcbrtentry(in6p);
 			if (rt == NULL)
@@ -1851,6 +1855,7 @@ tcp6_mtudisc(struct in6pcb *in6p, int er
 			tp->snd_cwnd = TCP_INITIAL_WINDOW(tcp_init_win,
 			    rt->rt_rmx.rmx_mtu);
 		}
+		in6_pcbrtentry_unref(rt, in6p);
 	}
 
 	/*
@@ -2039,6 +2044,16 @@ tcp_mss_from_peer(struct tcpcb *tp, int 
 		tp->snd_ssthresh = max(2 * mss, rt->rt_rmx.rmx_ssthresh);
 	}
 #endif
+#if defined(RTV_SPIPE) || defined(RTV_SSTHRESH)
+#ifdef INET
+	if (tp->t_inpcb)
+		in_pcbrtentry_unref(rt, tp->t_inpcb);
+#endif
+#ifdef INET6
+	if (tp->t_in6pcb)
+		in6_pcbrtentry_unref(rt, tp->t_in6pcb);
+#endif
+#endif
 }
 
 /*
@@ -2135,6 +2150,16 @@ tcp_established(struct tcpcb *tp)
 			bufsize = sb_max;
 		(void) sbreserve(&so->so_rcv, bufsize, so);
 	}
+#ifdef RTV_RPIPE
+#ifdef INET
+	if (tp->t_inpcb)
+		in_pcbrtentry_unref(rt, tp->t_inpcb);
+#endif
+#ifdef INET6
+	if (tp->t_in6pcb)
+		in6_pcbrtentry_unref(rt, tp->t_in6pcb);
+#endif
+#endif
 }
 
 /*
@@ -2188,6 +2213,14 @@ tcp_rmx_rtt(struct tcpcb *tp)
 		    ((tp->t_srtt >> 2) + tp->t_rttvar) >> (1 + 2),
 		    tp->t_rttmin, TCPTV_REXMTMAX);
 	}
+#ifdef INET
+	if (tp->t_inpcb)
+		in_pcbrtentry_unref(rt, tp->t_inpcb);
+#endif
+#ifdef INET6
+	if (tp->t_in6pcb)
+		in6_pcbrtentry_unref(rt, tp->t_in6pcb);
+#endif
 #endif
 }
 

Index: src/sys/netinet6/frag6.c
diff -u src/sys/netinet6/frag6.c:1.57 src/sys/netinet6/frag6.c:1.58
--- src/sys/netinet6/frag6.c:1.57	Wed Nov  9 03:49:38 2016
+++ src/sys/netinet6/frag6.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: frag6.c,v 1.57 2016/11/09 03:49:38 ozaki-r Exp $	*/
+/*	$NetBSD: frag6.c,v 1.58 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: frag6.c,v 1.40 2002/05/27 21:40:31 itojun Exp $	*/
 
 /*
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: frag6.c,v 1.57 2016/11/09 03:49:38 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: frag6.c,v 1.58 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -182,6 +182,7 @@ frag6_input(struct mbuf **mp, int *offp,
 		IP6_STATINC(IP6_STAT_REASSEMBLED);
 		in6_ifstat_inc(dstifp, ifs6_reass_ok);
 		*offp = offset;
+		rtcache_unref(rt, &ro);
 		return ip6f->ip6f_nxt;		
 	}
 
@@ -463,6 +464,7 @@ insert:
 
 	IP6_STATINC(IP6_STAT_REASSEMBLED);
 	in6_ifstat_inc(dstifp, ifs6_reass_ok);
+	rtcache_unref(rt, &ro);
 
 	/*
 	 * Tell launch routine the next header
@@ -480,6 +482,7 @@ insert:
 	IP6_STATINC(IP6_STAT_FRAGDROPPED);
 	m_freem(m);
  done:
+	rtcache_unref(rt, &ro);
 	return IPPROTO_DONE;
 }
 

Index: src/sys/netinet6/in6_gif.c
diff -u src/sys/netinet6/in6_gif.c:1.79 src/sys/netinet6/in6_gif.c:1.80
--- src/sys/netinet6/in6_gif.c:1.79	Fri Jul 15 07:40:09 2016
+++ src/sys/netinet6/in6_gif.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in6_gif.c,v 1.79 2016/07/15 07:40:09 ozaki-r Exp $	*/
+/*	$NetBSD: in6_gif.c,v 1.80 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: in6_gif.c,v 1.62 2001/07/29 04:27:25 itojun Exp $	*/
 
 /*
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in6_gif.c,v 1.79 2016/07/15 07:40:09 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in6_gif.c,v 1.80 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -176,17 +176,20 @@ in6_gif_output(struct ifnet *ifp, int fa
 	ip6->ip6_flow |= htonl((u_int32_t)otos << 20);
 
 	sockaddr_in6_init(&u.dst6, &sin6_dst->sin6_addr, 0, 0, 0);
-	if ((rt = rtcache_lookup(&sc->gif_ro, &u.dst)) == NULL) {
+	rt = rtcache_lookup(&sc->gif_ro, &u.dst);
+	if (rt == NULL) {
 		m_freem(m);
 		return ENETUNREACH;
 	}
 
 	/* If the route constitutes infinite encapsulation, punt. */
 	if (rt->rt_ifp == ifp) {
+		rtcache_unref(rt, &sc->gif_ro);
 		rtcache_free(&sc->gif_ro);
 		m_freem(m);
 		return ENETUNREACH;	/* XXX */
 	}
+	rtcache_unref(rt, &sc->gif_ro);
 
 #ifdef IPV6_MINMTU
 	/*

Index: src/sys/netinet6/in6_pcb.c
diff -u src/sys/netinet6/in6_pcb.c:1.152 src/sys/netinet6/in6_pcb.c:1.153
--- src/sys/netinet6/in6_pcb.c:1.152	Mon Oct 31 14:34:32 2016
+++ src/sys/netinet6/in6_pcb.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in6_pcb.c,v 1.152 2016/10/31 14:34:32 christos Exp $	*/
+/*	$NetBSD: in6_pcb.c,v 1.153 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: in6_pcb.c,v 1.84 2001/02/08 18:02:08 itojun Exp $	*/
 
 /*
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in6_pcb.c,v 1.152 2016/10/31 14:34:32 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in6_pcb.c,v 1.153 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -696,7 +696,6 @@ in6_pcbnotify(struct inpcbtable *table, 
     u_int fport_arg, const struct sockaddr *src, u_int lport_arg, int cmd,
     void *cmdarg, void (*notify)(struct in6pcb *, int))
 {
-	struct rtentry *rt;
 	struct inpcb_hdr *inph, *ninph;
 	struct sockaddr_in6 sa6_src;
 	const struct sockaddr_in6 *sa6_dst;
@@ -738,6 +737,8 @@ in6_pcbnotify(struct inpcbtable *table, 
 	errno = inet6ctlerrmap[cmd];
 	TAILQ_FOREACH_SAFE(inph, &table->inpt_queue, inph_queue, ninph) {
 		struct in6pcb *in6p = (struct in6pcb *)inph;
+		struct rtentry *rt = NULL;
+
 		if (in6p->in6p_af != AF_INET6)
 			continue;
 
@@ -783,9 +784,12 @@ in6_pcbnotify(struct inpcbtable *table, 
 			if (dst6 == NULL)
 				;
 			else if (IN6_ARE_ADDR_EQUAL(&dst6->sin6_addr,
-			    &sa6_dst->sin6_addr))
+			    &sa6_dst->sin6_addr)) {
+				rtcache_unref(rt, &in6p->in6p_route);
 				goto do_notify;
+			}
 		}
+		rtcache_unref(rt, &in6p->in6p_route);
 
 		/*
 		 * If the error designates a new path MTU for a destination
@@ -887,8 +891,11 @@ in6_pcbpurgeif(struct inpcbtable *table,
 		if (in6p->in6p_af != AF_INET6)
 			continue;
 		if ((rt = rtcache_validate(&in6p->in6p_route)) != NULL &&
-		    rt->rt_ifp == ifp)
+		    rt->rt_ifp == ifp) {
+			rtcache_unref(rt, &in6p->in6p_route);
 			in6_rtchange(in6p, 0);
+		} else
+			rtcache_unref(rt, &in6p->in6p_route);
 	}
 }
 
@@ -916,9 +923,16 @@ in6_losing(struct in6pcb *in6p)
 	info.rti_info[RTAX_NETMASK] = rt_mask(rt);
 	rt_missmsg(RTM_LOSING, &info, rt->rt_flags, 0);
 	if (rt->rt_flags & RTF_DYNAMIC) {
-		(void)rtrequest(RTM_DELETE, rt_getkey(rt),
-		    rt->rt_gateway, rt_mask(rt), rt->rt_flags, NULL);
-	}
+		int error;
+		struct rtentry *nrt;
+
+		error = rtrequest(RTM_DELETE, rt_getkey(rt),
+		    rt->rt_gateway, rt_mask(rt), rt->rt_flags, &nrt);
+		rtcache_unref(rt, &in6p->in6p_route);
+		if (error == 0)
+			rtfree(nrt);
+	} else
+		rtcache_unref(rt, &in6p->in6p_route);
 	/*
 	 * A new route can be allocated
 	 * the next time output is attempted.
@@ -1151,6 +1165,13 @@ in6_pcbrtentry(struct in6pcb *in6p)
 	return rt;
 }
 
+void
+in6_pcbrtentry_unref(struct rtentry *rt, struct in6pcb *in6p)
+{
+
+	rtcache_unref(rt, &in6p->in6p_route);
+}
+
 struct in6pcb *
 in6_pcblookup_connect(struct inpcbtable *table, const struct in6_addr *faddr6,
 		      u_int fport_arg, const struct in6_addr *laddr6, u_int lport_arg,

Index: src/sys/netinet6/in6_pcb.h
diff -u src/sys/netinet6/in6_pcb.h:1.46 src/sys/netinet6/in6_pcb.h:1.47
--- src/sys/netinet6/in6_pcb.h:1.46	Sun May 24 15:43:45 2015
+++ src/sys/netinet6/in6_pcb.h	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in6_pcb.h,v 1.46 2015/05/24 15:43:45 rtr Exp $	*/
+/*	$NetBSD: in6_pcb.h,v 1.47 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: in6_pcb.h,v 1.45 2001/02/09 05:59:46 itojun Exp $	*/
 
 /*
@@ -179,6 +179,8 @@ int	in6_pcbsetport(struct sockaddr_in6 *
 
 extern struct rtentry *
 	in6_pcbrtentry(struct in6pcb *);
+extern void
+	in6_pcbrtentry_unref(struct rtentry *, struct in6pcb *);
 extern struct in6pcb *in6_pcblookup_connect(struct inpcbtable *,
 					    const struct in6_addr *, u_int, const struct in6_addr *, u_int, int,
 					    struct vestigial_inpcb *);

Index: src/sys/netinet6/in6_src.c
diff -u src/sys/netinet6/in6_src.c:1.75 src/sys/netinet6/in6_src.c:1.76
--- src/sys/netinet6/in6_src.c:1.75	Fri Dec  2 00:19:54 2016
+++ src/sys/netinet6/in6_src.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: in6_src.c,v 1.75 2016/12/02 00:19:54 ozaki-r Exp $	*/
+/*	$NetBSD: in6_src.c,v 1.76 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: in6_src.c,v 1.159 2005/10/19 01:40:32 t-momose Exp $	*/
 
 /*
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: in6_src.c,v 1.75 2016/12/02 00:19:54 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: in6_src.c,v 1.76 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -599,6 +599,7 @@ in6_selectroute(struct sockaddr_in6 *dst
 	} u;
 
 	KASSERT(ro != NULL);
+	KASSERT(*ro != NULL);
 	KASSERT(retrt != NULL);
 
 #if 0
@@ -638,10 +639,14 @@ in6_selectroute(struct sockaddr_in6 *dst
 		rt = rtcache_lookup(ron, sin6tosa(sin6_next));
 		if (rt == NULL || (rt->rt_flags & RTF_GATEWAY) != 0 ||
 		    !nd6_is_addr_neighbor(sin6_next, rt->rt_ifp)) {
+			if (rt != NULL) {
+				if (count_discard)
+					in6_ifstat_inc(rt->rt_ifp,
+					    ifs6_out_discard);
+				rtcache_unref(rt, ron);
+				rt = NULL;
+			}
 			rtcache_free(ron);
-			if (rt != NULL && count_discard)
-				in6_ifstat_inc(rt->rt_ifp, ifs6_out_discard);
-			rt = NULL;
 			error = EHOSTUNREACH;
 			goto done;
 		}
@@ -691,7 +696,7 @@ in6_selectif(struct sockaddr_in6 *dstsoc
 	struct ip6_moptions *mopts, struct route *ro, struct ifnet **retifp,
 	struct psref *psref)
 {
-	int error;
+	int error = 0;
 	struct rtentry *rt = NULL;
 	struct in6_addr *dst;
 	struct in6_pktinfo *pi = NULL;
@@ -743,8 +748,11 @@ getroute:
 	 * Although this may not be very harmful, it should still be confusing.
 	 * We thus reject the case here.
 	 */
-	if ((rt->rt_flags & (RTF_REJECT | RTF_BLACKHOLE)))
-		return (rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
+	if ((rt->rt_flags & (RTF_REJECT | RTF_BLACKHOLE))) {
+		error = (rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
+		/* XXX: ifp can be returned with psref even if error */
+		goto out;
+	}
 
 	/*
 	 * Adjust the "outgoing" interface.  If we're going to loop the packet
@@ -760,8 +768,9 @@ getroute:
 		*retifp = rt->rt_ifa->ifa_ifp;
 		if_acquire_NOMPSAFE(*retifp, psref);
 	}
-
-	return (0);
+out:
+	rtcache_unref(rt, ro);
+	return error;
 }
 
 /*
@@ -791,9 +800,11 @@ in6_selecthlim_rt(struct in6pcb *in6p)
 		return in6_selecthlim(in6p, NULL);
 
 	rt = rtcache_validate(&in6p->in6p_route);
-	if (rt != NULL)
-		return in6_selecthlim(in6p, rt->rt_ifp);
-	else
+	if (rt != NULL) {
+		int ret = in6_selecthlim(in6p, rt->rt_ifp);
+		rtcache_unref(rt, &in6p->in6p_route);
+		return ret;
+	} else
 		return in6_selecthlim(in6p, NULL);
 }
 

Index: src/sys/netinet6/ip6_etherip.c
diff -u src/sys/netinet6/ip6_etherip.c:1.18 src/sys/netinet6/ip6_etherip.c:1.19
--- src/sys/netinet6/ip6_etherip.c:1.18	Fri Jun 10 13:27:16 2016
+++ src/sys/netinet6/ip6_etherip.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*      $NetBSD: ip6_etherip.c,v 1.18 2016/06/10 13:27:16 ozaki-r Exp $        */
+/*      $NetBSD: ip6_etherip.c,v 1.19 2016/12/08 05:16:34 ozaki-r Exp $        */
 
 /*
  *  Copyright (c) 2006, Hans Rosenfeld <rosenf...@grumpf.hope-2000.org>
@@ -58,7 +58,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_etherip.c,v 1.18 2016/06/10 13:27:16 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_etherip.c,v 1.19 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -172,10 +172,12 @@ ip6_etherip_output(struct ifnet *ifp, st
 	}
 	/* if it constitutes infinite encapsulation, punt. */
 	if (rt->rt_ifp == ifp) {
+		rtcache_unref(rt, &sc->sc_ro);
 		rtcache_free(&sc->sc_ro);
 		m_freem(m);
 		return ENETUNREACH;     /* XXX */
 	}
+	rtcache_unref(rt, &sc->sc_ro);
 
 	/*
 	 * force fragmentation to minimum MTU, to avoid path MTU discovery.

Index: src/sys/netinet6/ip6_flow.c
diff -u src/sys/netinet6/ip6_flow.c:1.32 src/sys/netinet6/ip6_flow.c:1.33
--- src/sys/netinet6/ip6_flow.c:1.32	Tue Oct 18 07:30:31 2016
+++ src/sys/netinet6/ip6_flow.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip6_flow.c,v 1.32 2016/10/18 07:30:31 ozaki-r Exp $	*/
+/*	$NetBSD: ip6_flow.c,v 1.33 2016/12/08 05:16:34 ozaki-r Exp $	*/
 
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_flow.c,v 1.32 2016/10/18 07:30:31 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_flow.c,v 1.33 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_net_mpsafe.h"
@@ -259,7 +259,7 @@ ip6flow_fastforward(struct mbuf **mp)
 {
 	struct ip6flow *ip6f;
 	struct ip6_hdr *ip6;
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 	struct mbuf *m;
 	const struct sockaddr *dst;
 	int error;
@@ -327,14 +327,14 @@ ip6flow_fastforward(struct mbuf **mp)
 	if ((rt = rtcache_validate(&ip6f->ip6f_ro)) == NULL ||
 	    (rt->rt_ifp->if_flags & IFF_UP) == 0 ||
 	    (rt->rt_flags & RTF_BLACKHOLE) != 0)
-		goto out;
+		goto out_unref;
 
 	/*
 	 * Packet size greater than MTU?
 	 */
 	if (m->m_pkthdr.len > rt->rt_ifp->if_mtu) {
 		/* Return to main IPv6 input function. */
-		goto out;
+		goto out_unref;
 	}
 
 	/*
@@ -343,7 +343,7 @@ ip6flow_fastforward(struct mbuf **mp)
 	m->m_pkthdr.csum_flags = 0;
 
 	if (ip6->ip6_hlim <= IPV6_HLIMDEC)
-		goto out;
+		goto out_unref;
 
 	/* Decrement hop limit (same as TTL) */
 	ip6->ip6_hlim -= IPV6_HLIMDEC;
@@ -373,7 +373,9 @@ ip6flow_fastforward(struct mbuf **mp)
 		ip6f->ip6f_forwarded++;
 	}
 	ret = 1;
- out:
+out_unref:
+	rtcache_unref(rt, &ip6f->ip6f_ro);
+out:
 	mutex_exit(&ip6flow_lock);
 	return ret;
 }
@@ -382,12 +384,11 @@ ip6flow_fastforward(struct mbuf **mp)
  * Add the IPv6 flow statistics to the main IPv6 statistics.
  */
 static void
-ip6flow_addstats(const struct ip6flow *ip6f)
+ip6flow_addstats_rt(struct rtentry *rt, struct ip6flow *ip6f)
 {
-	struct rtentry *rt;
 	uint64_t *ip6s;
 
-	if ((rt = rtcache_validate(&ip6f->ip6f_ro)) != NULL)
+	if (rt != NULL)
 		rt->rt_use += ip6f->ip6f_uses;
 	ip6s = IP6_STAT_GETREF();
 	ip6s[IP6_STAT_FASTFORWARDFLOWS] = ip6flow_inuse;
@@ -399,6 +400,16 @@ ip6flow_addstats(const struct ip6flow *i
 	IP6_STAT_PUTREF();
 }
 
+static void
+ip6flow_addstats(struct ip6flow *ip6f)
+{
+	struct rtentry *rt;
+
+	rt = rtcache_validate(&ip6f->ip6f_ro);
+	ip6flow_addstats_rt(rt, ip6f);
+	rtcache_unref(rt, &ip6f->ip6f_ro);
+}
+
 /*
  * Add statistics and free the flow.
  */
@@ -452,12 +463,14 @@ ip6flow_reap_locked(int just_one)
 		struct ip6flow *maybe_ip6f = TAILQ_LAST(&ip6flowlist, ip6flowhead);
 
 		TAILQ_FOREACH(ip6f, &ip6flowlist, ip6f_list) {
+			struct rtentry *rt;
 			/*
 			 * If this no longer points to a valid route -
 			 * reclaim it.
 			 */
-			if (rtcache_validate(&ip6f->ip6f_ro) == NULL)
+			if ((rt = rtcache_validate(&ip6f->ip6f_ro)) == NULL)
 				goto done;
+			rtcache_unref(rt, &ip6f->ip6f_ro);
 			/*
 			 * choose the one that's been least recently
 			 * used or has had the least uses in the
@@ -516,17 +529,19 @@ ip6flow_slowtimo_work(struct work *wk, v
 	mutex_enter(&ip6flow_lock);
 
 	for (ip6f = TAILQ_FIRST(&ip6flowlist); ip6f != NULL; ip6f = next_ip6f) {
+		struct rtentry *rt = NULL;
 		next_ip6f = TAILQ_NEXT(ip6f, ip6f_list);
 		if (PRT_SLOW_ISEXPIRED(ip6f->ip6f_timer) ||
-		    rtcache_validate(&ip6f->ip6f_ro) == NULL) {
+		    (rt = rtcache_validate(&ip6f->ip6f_ro)) == NULL) {
 			ip6flow_free(ip6f);
 		} else {
 			ip6f->ip6f_last_uses = ip6f->ip6f_uses;
-			ip6flow_addstats(ip6f);
+			ip6flow_addstats_rt(rt, ip6f);
 			ip6f->ip6f_uses = 0;
 			ip6f->ip6f_dropped = 0;
 			ip6f->ip6f_forwarded = 0;
 		}
+		rtcache_unref(rt, &ip6f->ip6f_ro);
 	}
 
 	mutex_exit(&ip6flow_lock);
@@ -552,7 +567,7 @@ ip6flow_slowtimo(void)
  * IPv6 stack. Now create/update a flow.
  */
 void
-ip6flow_create(const struct route *ro, struct mbuf *m)
+ip6flow_create(struct route *ro, struct mbuf *m)
 {
 	const struct ip6_hdr *ip6;
 	struct ip6flow *ip6f;

Index: src/sys/netinet6/ip6_forward.c
diff -u src/sys/netinet6/ip6_forward.c:1.81 src/sys/netinet6/ip6_forward.c:1.82
--- src/sys/netinet6/ip6_forward.c:1.81	Wed Aug 31 09:14:47 2016
+++ src/sys/netinet6/ip6_forward.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $	*/
+/*	$NetBSD: ip6_forward.c,v 1.82 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $	*/
 
 /*
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.82 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_gateway.h"
@@ -128,7 +128,7 @@ ip6_forward(struct mbuf *m, int srcrt)
 {
 	struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
 	const struct sockaddr_in6 *dst;
-	struct rtentry *rt;
+	struct rtentry *rt = NULL;
 	int error = 0, type = 0, code = 0;
 	struct mbuf *mcopy = NULL;
 	struct ifnet *origifp;	/* maybe unnecessary */
@@ -136,7 +136,7 @@ ip6_forward(struct mbuf *m, int srcrt)
 	struct in6_addr src_in6, dst_in6;
 	struct ifnet *rcvif = NULL;
 	struct psref psref;
-	struct route *ro;
+	struct route *ro = NULL;
 #ifdef IPSEC
 	int needipsec = 0;
 	struct secpolicy *sp = NULL;
@@ -213,14 +213,14 @@ ip6_forward(struct mbuf *m, int srcrt)
 		} u;
 
 		sockaddr_in6_init(&u.dst6, &ip6->ip6_dst, 0, 0, 0);
-		if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
+		rt = rtcache_lookup(ro, &u.dst);
+		if (rt == NULL) {
 			IP6_STATINC(IP6_STAT_NOROUTE);
 			/* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_noroute) */
 			if (mcopy) {
 				icmp6_error(mcopy, ICMP6_DST_UNREACH,
 					    ICMP6_DST_UNREACH_NOROUTE, 0);
 			}
-			percpu_putref(ip6_forward_rt_percpu);
 			goto drop;
 		}
 	} else if ((rt = rtcache_validate(ro)) == NULL &&
@@ -235,11 +235,9 @@ ip6_forward(struct mbuf *m, int srcrt)
 			icmp6_error(mcopy, ICMP6_DST_UNREACH,
 			    ICMP6_DST_UNREACH_NOROUTE, 0);
 		}
-		percpu_putref(ip6_forward_rt_percpu);
 		goto drop;
 	}
 	dst = satocsin6(rtcache_getdst(ro));
-	percpu_putref(ip6_forward_rt_percpu);
 
 	/*
 	 * Source scope check: if a packet can't be delivered to its
@@ -324,11 +322,9 @@ ip6_forward(struct mbuf *m, int srcrt)
 	 */
 	if (rt->rt_ifp == rcvif && !srcrt && ip6_sendredirects &&
 	    (rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
-		ro = percpu_getref(ip6_forward_rt_percpu);
 		if ((rt->rt_ifp->if_flags & IFF_POINTOPOINT) &&
 		    nd6_is_addr_neighbor(satocsin6(rtcache_getdst(ro)),
 		                         rt->rt_ifp)) {
-			percpu_putref(ip6_forward_rt_percpu);
 			/*
 			 * If the incoming interface is equal to the outgoing
 			 * one, the link attached to the interface is
@@ -348,7 +344,6 @@ ip6_forward(struct mbuf *m, int srcrt)
 				    ICMP6_DST_UNREACH_ADDR, 0);
 			goto drop;
 		}
-		percpu_putref(ip6_forward_rt_percpu);
 		type = ND_REDIRECT;
 	}
 
@@ -417,10 +412,11 @@ ip6_forward(struct mbuf *m, int srcrt)
 			IP6_STATINC(IP6_STAT_REDIRECTSENT);
 		else {
 #ifdef GATEWAY
-			ro = percpu_getref(ip6_forward_rt_percpu);
+			/* Need to release rt here */
+			rtcache_unref(rt, ro);
+			rt = NULL;
 			if (m->m_flags & M_CANFASTFWD)
 				ip6flow_create(ro, m);
-			percpu_putref(ip6_forward_rt_percpu);
 #endif
 			if (mcopy)
 				goto freecopy;
@@ -464,6 +460,9 @@ ip6_forward(struct mbuf *m, int srcrt)
  drop:
  	m_freem(m);
  out:
+	rtcache_unref(rt, ro);
+	if (ro != NULL)
+		percpu_putref(ip6_forward_rt_percpu);
 	if (rcvif != NULL)
 		m_put_rcvif_psref(rcvif, &psref);
 	return;

Index: src/sys/netinet6/ip6_input.c
diff -u src/sys/netinet6/ip6_input.c:1.170 src/sys/netinet6/ip6_input.c:1.171
--- src/sys/netinet6/ip6_input.c:1.170	Tue Nov  1 10:32:57 2016
+++ src/sys/netinet6/ip6_input.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip6_input.c,v 1.170 2016/11/01 10:32:57 ozaki-r Exp $	*/
+/*	$NetBSD: ip6_input.c,v 1.171 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $	*/
 
 /*
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.170 2016/11/01 10:32:57 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.171 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_gateway.h"
@@ -269,7 +269,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 	int nxt, ours = 0, rh_present = 0;
 	struct ifnet *deliverifp = NULL;
 	int srcrt = 0;
-	const struct rtentry *rt;
+	struct rtentry *rt = NULL;
 	union {
 		struct sockaddr		dst;
 		struct sockaddr_in6	dst6;
@@ -454,6 +454,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 		goto bad;
 	}
 
+	ro = percpu_getref(ip6_forward_rt_percpu);
 	/*
 	 * Multicast check
 	 */
@@ -474,7 +475,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 			ip6s[IP6_STAT_CANTFORWARD]++;
 			IP6_STAT_PUTREF();
 			in6_ifstat_inc(rcvif, ifs6_in_discard);
-			goto bad;
+			goto bad_unref;
 		}
 		deliverifp = rcvif;
 		goto hbhcheck;
@@ -485,9 +486,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 	/*
 	 *  Unicast check
 	 */
-	ro = percpu_getref(ip6_forward_rt_percpu);
 	rt = rtcache_lookup2(ro, &u.dst, 1, &hit);
-	percpu_putref(ip6_forward_rt_percpu);
 	if (hit)
 		IP6_STATINC(IP6_STAT_FORWARD_CACHEHIT);
 	else
@@ -533,7 +532,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 			    ip6_sprintf(&ip6->ip6_src),
 			    ip6_sprintf(&ip6->ip6_dst));
 
-			goto bad;
+			goto bad_unref;
 		}
 	}
 
@@ -579,7 +578,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 	if (!ip6_forwarding) {
 		IP6_STATINC(IP6_STAT_CANTFORWARD);
 		in6_ifstat_inc(rcvif, ifs6_in_discard);
-		goto bad;
+		goto bad_unref;
 	}
 
   hbhcheck:
@@ -618,6 +617,8 @@ ip6_input(struct mbuf *m, struct ifnet *
 #if 0	/*touches NULL pointer*/
 			in6_ifstat_inc(rcvif, ifs6_in_discard);
 #endif
+			rtcache_unref(rt, ro);
+			percpu_putref(ip6_forward_rt_percpu);
 			return;	/* m have already been freed */
 		}
 
@@ -641,12 +642,16 @@ ip6_input(struct mbuf *m, struct ifnet *
 			icmp6_error(m, ICMP6_PARAM_PROB,
 				    ICMP6_PARAMPROB_HEADER,
 				    (char *)&ip6->ip6_plen - (char *)ip6);
+			rtcache_unref(rt, ro);
+			percpu_putref(ip6_forward_rt_percpu);
 			return;
 		}
 		IP6_EXTHDR_GET(hbh, struct ip6_hbh *, m, sizeof(struct ip6_hdr),
 			sizeof(struct ip6_hbh));
 		if (hbh == NULL) {
 			IP6_STATINC(IP6_STAT_TOOSHORT);
+			rtcache_unref(rt, ro);
+			percpu_putref(ip6_forward_rt_percpu);
 			return;
 		}
 		KASSERT(IP6_HDR_ALIGNED_P(hbh));
@@ -670,7 +675,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 	if (m->m_pkthdr.len - sizeof(struct ip6_hdr) < plen) {
 		IP6_STATINC(IP6_STAT_TOOSHORT);
 		in6_ifstat_inc(rcvif, ifs6_in_truncated);
-		goto bad;
+		goto bad_unref;
 	}
 	if (m->m_pkthdr.len > sizeof(struct ip6_hdr) + plen) {
 		if (m->m_len == m->m_pkthdr.len) {
@@ -700,13 +705,17 @@ ip6_input(struct mbuf *m, struct ifnet *
 			SOFTNET_UNLOCK();
 
 			if (error != 0) {
+				rtcache_unref(rt, ro);
+				percpu_putref(ip6_forward_rt_percpu);
 				IP6_STATINC(IP6_STAT_CANTFORWARD);
 				goto bad;
 			}
 		}
 		if (!ours)
-			goto bad;
+			goto bad_unref;
 	} else if (!ours) {
+		rtcache_unref(rt, ro);
+		percpu_putref(ip6_forward_rt_percpu);
 		ip6_forward(m, srcrt);
 		return;
 	}
@@ -726,7 +735,7 @@ ip6_input(struct mbuf *m, struct ifnet *
 	    IN6_IS_ADDR_V4MAPPED(&ip6->ip6_dst)) {
 		IP6_STATINC(IP6_STAT_BADSCOPE);
 		in6_ifstat_inc(rcvif, ifs6_in_addrerr);
-		goto bad;
+		goto bad_unref;
 	}
 
 	/*
@@ -746,6 +755,12 @@ ip6_input(struct mbuf *m, struct ifnet *
 	in6_ifstat_inc(deliverifp, ifs6_in_deliver);
 	nest = 0;
 
+	if (rt != NULL) {
+		rtcache_unref(rt, ro);
+		rt = NULL;
+	}
+	percpu_putref(ip6_forward_rt_percpu);
+
 	rh_present = 0;
 	while (nxt != IPPROTO_DONE) {
 		if (ip6_hdrnestlimit && (++nest > ip6_hdrnestlimit)) {
@@ -797,8 +812,13 @@ ip6_input(struct mbuf *m, struct ifnet *
 		SOFTNET_UNLOCK();
 	}
 	return;
+
+ bad_unref:
+	rtcache_unref(rt, ro);
+	percpu_putref(ip6_forward_rt_percpu);
  bad:
 	m_freem(m);
+	return;
 }
 
 /*

Index: src/sys/netinet6/ip6_output.c
diff -u src/sys/netinet6/ip6_output.c:1.178 src/sys/netinet6/ip6_output.c:1.179
--- src/sys/netinet6/ip6_output.c:1.178	Thu Nov 10 04:13:53 2016
+++ src/sys/netinet6/ip6_output.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip6_output.c,v 1.178 2016/11/10 04:13:53 ozaki-r Exp $	*/
+/*	$NetBSD: ip6_output.c,v 1.179 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: ip6_output.c,v 1.172 2001/03/25 09:55:56 itojun Exp $	*/
 
 /*
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ip6_output.c,v 1.178 2016/11/10 04:13:53 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ip6_output.c,v 1.179 2016/12/08 05:16:34 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
@@ -695,8 +695,10 @@ ip6_output(
 		sockaddr_in6_init(&u.dst6, &finaldst, 0, 0, 0);
 		rt_pmtu = rtcache_lookup(ro_pmtu, &u.dst);
 	} else
-		rt_pmtu = rtcache_validate(ro_pmtu);
+		rt_pmtu = rt;
 	error = ip6_getpmtu(rt_pmtu, ifp, &mtu, &alwaysfrag);
+	if (rt_pmtu != NULL && rt_pmtu != rt)
+		rtcache_unref(rt_pmtu, ro_pmtu);
 	if (error != 0)
 		goto bad;
 
@@ -1037,6 +1039,7 @@ sendorfree:
 		IP6_STATINC(IP6_STAT_FRAGMENTED);
 
 done:
+	rtcache_unref(rt, ro);
 	if (ro == &ip6route)
 		rtcache_free(&ip6route);
 
@@ -1869,6 +1872,7 @@ else 					\
 			sockaddr_in6_init(&u.dst6, &in6p->in6p_faddr, 0, 0, 0);
 			rt = rtcache_lookup(ro, &u.dst);
 			error = ip6_getpmtu(rt, NULL, &pmtu, NULL);
+			rtcache_unref(rt, ro);
 			if (error)
 				break;
 			if (pmtu > IPV6_MAXPACKET)
@@ -2405,7 +2409,10 @@ ip6_get_membership(const struct sockopt 
 		error = rtcache_setdst(&ro, &u.dst);
 		if (error != 0)
 			return error;
-		*ifp = (rt = rtcache_init(&ro)) != NULL ? rt->rt_ifp : NULL;
+		rt = rtcache_init(&ro);
+		*ifp = rt != NULL ? rt->rt_ifp : NULL;
+		/* FIXME *ifp is NOMPSAFE */
+		rtcache_unref(rt, &ro);
 		rtcache_free(&ro);
 	} else {
 		/*

Index: src/sys/netinet6/ip6_var.h
diff -u src/sys/netinet6/ip6_var.h:1.70 src/sys/netinet6/ip6_var.h:1.71
--- src/sys/netinet6/ip6_var.h:1.70	Thu Nov 10 04:13:53 2016
+++ src/sys/netinet6/ip6_var.h	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ip6_var.h,v 1.70 2016/11/10 04:13:53 ozaki-r Exp $	*/
+/*	$NetBSD: ip6_var.h,v 1.71 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$KAME: ip6_var.h,v 1.33 2000/06/11 14:59:20 jinmei Exp $	*/
 
 /*
@@ -377,7 +377,7 @@ void	frag6_drainstub(void);
 int	ip6flow_init(int);
 void	ip6flow_poolinit(void);
 struct  ip6flow *ip6flow_reap(int);
-void    ip6flow_create(const struct route *, struct mbuf *);
+void    ip6flow_create(struct route *, struct mbuf *);
 void    ip6flow_slowtimo(void);
 int	ip6flow_invalidate_all(int);
 

Index: src/sys/netipsec/ipsec.c
diff -u src/sys/netipsec/ipsec.c:1.66 src/sys/netipsec/ipsec.c:1.67
--- src/sys/netipsec/ipsec.c:1.66	Wed Apr  1 02:49:44 2015
+++ src/sys/netipsec/ipsec.c	Thu Dec  8 05:16:34 2016
@@ -1,4 +1,4 @@
-/*	$NetBSD: ipsec.c,v 1.66 2015/04/01 02:49:44 ozaki-r Exp $	*/
+/*	$NetBSD: ipsec.c,v 1.67 2016/12/08 05:16:34 ozaki-r Exp $	*/
 /*	$FreeBSD: /usr/local/www/cvsroot/FreeBSD/src/sys/netipsec/ipsec.c,v 1.2.2.2 2003/07/01 01:38:13 sam Exp $	*/
 /*	$KAME: ipsec.c,v 1.103 2001/05/24 07:14:18 sakane Exp $	*/
 
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipsec.c,v 1.66 2015/04/01 02:49:44 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipsec.c,v 1.67 2016/12/08 05:16:34 ozaki-r Exp $");
 
 /*
  * IPsec controller part.
@@ -929,6 +929,7 @@ ipsec4_forward(struct mbuf *m, int *dest
 			    rt->rt_rmx.rmx_mtu : rt->rt_ifp->if_mtu;
 			*destmtu -= ipsechdr;
 		}
+		rtcache_unref(rt, ro);
 	}
 	KEY_FREESP(&sp);
 	return 0;

Reply via email to