> On 3 Sep 2022, at 23:47, Alexander Bluhm <alexander.bl...@gmx.net> wrote: > > Hi, > > The next small step towards parallel network stack is to use shared > netlock in soreceive(). The UDP and IP divert layer provide locking > of the PCB. If that is possible, use shared instead of exclusive > netlock in soreceive(). The PCB mutex provides a per socket lock > against multiple soreceive() running in parallel. > > The solock_shared() is a bit hacky. Especially where we have to > release and regrab both locks in sosleep_nsec(). But it does what > we need. > > The udp_input() keeps exclusive lock for now. Socket splicing and > IP header chains are not completely MP safe yet. Also udp_input() > has some tentacles that need more testing. > > Ok to procceed with soreceive() unlocking before release? I think > udp_input() has to wait post 7.2. >
I have no objections. > bluhm > > Index: kern/uipc_socket.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket.c,v > retrieving revision 1.287 > diff -u -p -r1.287 uipc_socket.c > --- kern/uipc_socket.c 3 Sep 2022 13:29:33 -0000 1.287 > +++ kern/uipc_socket.c 3 Sep 2022 19:23:43 -0000 > @@ -822,10 +822,10 @@ bad: > if (mp) > *mp = NULL; > > - solock(so); > + solock_shared(so); > restart: > if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) { > - sounlock(so); > + sounlock_shared(so); > return (error); > } > > @@ -893,7 +893,7 @@ restart: > sbunlock(so, &so->so_rcv); > error = sbwait(so, &so->so_rcv); > if (error) { > - sounlock(so); > + sounlock_shared(so); > return (error); > } > goto restart; > @@ -962,11 +962,11 @@ dontblock: > sbsync(&so->so_rcv, nextrecord); > if (controlp) { > if (pr->pr_domain->dom_externalize) { > - sounlock(so); > + sounlock_shared(so); > error = > (*pr->pr_domain->dom_externalize) > (cm, controllen, flags); > - solock(so); > + solock_shared(so); > } > *controlp = cm; > } else { > @@ -1040,9 +1040,9 @@ dontblock: > SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); > SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); > resid = uio->uio_resid; > - sounlock(so); > + sounlock_shared(so); > uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio); > - solock(so); > + solock_shared(so); > if (uio_error) > uio->uio_resid = resid - len; > } else > @@ -1126,7 +1126,7 @@ dontblock: > error = sbwait(so, &so->so_rcv); > if (error) { > sbunlock(so, &so->so_rcv); > - sounlock(so); > + sounlock_shared(so); > return (0); > } > if ((m = so->so_rcv.sb_mb) != NULL) > @@ -1171,7 +1171,7 @@ dontblock: > *flagsp |= flags; > release: > sbunlock(so, &so->so_rcv); > - sounlock(so); > + sounlock_shared(so); > return (error); > } > > Index: kern/uipc_socket2.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket2.c,v > retrieving revision 1.127 > diff -u -p -r1.127 uipc_socket2.c > --- kern/uipc_socket2.c 13 Aug 2022 21:01:46 -0000 1.127 > +++ kern/uipc_socket2.c 3 Sep 2022 19:23:43 -0000 > @@ -360,6 +360,24 @@ solock(struct socket *so) > } > } > > +void > +solock_shared(struct socket *so) > +{ > + switch (so->so_proto->pr_domain->dom_family) { > + case PF_INET: > + case PF_INET6: > + if (so->so_proto->pr_usrreqs->pru_lock != NULL) { > + NET_LOCK_SHARED(); > + pru_lock(so); > + } else > + NET_LOCK(); > + break; > + default: > + rw_enter_write(&so->so_lock); > + break; > + } > +} > + > int > solock_persocket(struct socket *so) > { > @@ -403,6 +421,24 @@ sounlock(struct socket *so) > } > > void > +sounlock_shared(struct socket *so) > +{ > + switch (so->so_proto->pr_domain->dom_family) { > + case PF_INET: > + case PF_INET6: > + if (so->so_proto->pr_usrreqs->pru_unlock != NULL) { > + pru_unlock(so); > + NET_UNLOCK_SHARED(); > + } else > + NET_UNLOCK(); > + break; > + default: > + rw_exit_write(&so->so_lock); > + break; > + } > +} > + > +void > soassertlocked(struct socket *so) > { > switch (so->so_proto->pr_domain->dom_family) { > @@ -425,7 +461,15 @@ sosleep_nsec(struct socket *so, void *id > switch (so->so_proto->pr_domain->dom_family) { > case PF_INET: > case PF_INET6: > + if (so->so_proto->pr_usrreqs->pru_unlock != NULL && > + rw_status(&netlock) == RW_READ) { > + pru_unlock(so); > + } > ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs); > + if (so->so_proto->pr_usrreqs->pru_lock != NULL && > + rw_status(&netlock) == RW_READ) { > + pru_lock(so); > + } > break; > default: > ret = rwsleep_nsec(ident, &so->so_lock, prio, wmesg, nsecs); > Index: netinet/ip_divert.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.c,v > retrieving revision 1.85 > diff -u -p -r1.85 ip_divert.c > --- netinet/ip_divert.c 3 Sep 2022 18:48:50 -0000 1.85 > +++ netinet/ip_divert.c 3 Sep 2022 19:23:43 -0000 > @@ -66,6 +66,8 @@ const struct pr_usrreqs divert_usrreqs = > .pru_usrreq = divert_usrreq, > .pru_attach = divert_attach, > .pru_detach = divert_detach, > + .pru_lock = divert_lock, > + .pru_unlock = divert_unlock, > .pru_bind = divert_bind, > .pru_shutdown = divert_shutdown, > .pru_send = divert_send, > @@ -325,6 +327,24 @@ divert_detach(struct socket *so) > > in_pcbdetach(inp); > return (0); > +} > + > +void > +divert_lock(struct socket *so) > +{ > + struct inpcb *inp = sotoinpcb(so); > + > + NET_ASSERT_LOCKED(); > + mtx_enter(&inp->inp_mtx); > +} > + > +void > +divert_unlock(struct socket *so) > +{ > + struct inpcb *inp = sotoinpcb(so); > + > + NET_ASSERT_LOCKED(); > + mtx_leave(&inp->inp_mtx); > } > > int > Index: netinet/ip_divert.h > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.h,v > retrieving revision 1.20 > diff -u -p -r1.20 ip_divert.h > --- netinet/ip_divert.h 28 Aug 2022 18:44:16 -0000 1.20 > +++ netinet/ip_divert.h 3 Sep 2022 19:23:43 -0000 > @@ -74,6 +74,8 @@ int divert_usrreq(struct socket *, > int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *); > int divert_attach(struct socket *, int); > int divert_detach(struct socket *); > +void divert_lock(struct socket *); > +void divert_unlock(struct socket *); > int divert_bind(struct socket *, struct mbuf *, struct proc *); > int divert_shutdown(struct socket *); > int divert_send(struct socket *, struct mbuf *, struct mbuf *, > Index: netinet/udp_usrreq.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_usrreq.c,v > retrieving revision 1.300 > diff -u -p -r1.300 udp_usrreq.c > --- netinet/udp_usrreq.c 3 Sep 2022 18:48:50 -0000 1.300 > +++ netinet/udp_usrreq.c 3 Sep 2022 19:23:43 -0000 > @@ -126,6 +126,8 @@ const struct pr_usrreqs udp_usrreqs = { > .pru_usrreq = udp_usrreq, > .pru_attach = udp_attach, > .pru_detach = udp_detach, > + .pru_lock = udp_lock, > + .pru_unlock = udp_unlock, > .pru_bind = udp_bind, > .pru_connect = udp_connect, > .pru_disconnect = udp_disconnect, > @@ -141,6 +143,8 @@ const struct pr_usrreqs udp6_usrreqs = { > .pru_usrreq = udp_usrreq, > .pru_attach = udp_attach, > .pru_detach = udp_detach, > + .pru_lock = udp_lock, > + .pru_unlock = udp_unlock, > .pru_bind = udp_bind, > .pru_connect = udp_connect, > .pru_disconnect = udp_disconnect, > @@ -1159,6 +1163,24 @@ udp_detach(struct socket *so) > > in_pcbdetach(inp); > return (0); > +} > + > +void > +udp_lock(struct socket *so) > +{ > + struct inpcb *inp = sotoinpcb(so); > + > + NET_ASSERT_LOCKED(); > + mtx_enter(&inp->inp_mtx); > +} > + > +void > +udp_unlock(struct socket *so) > +{ > + struct inpcb *inp = sotoinpcb(so); > + > + NET_ASSERT_LOCKED(); > + mtx_leave(&inp->inp_mtx); > } > > int > Index: netinet/udp_var.h > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_var.h,v > retrieving revision 1.45 > diff -u -p -r1.45 udp_var.h > --- netinet/udp_var.h 2 Sep 2022 13:12:32 -0000 1.45 > +++ netinet/udp_var.h 3 Sep 2022 19:23:43 -0000 > @@ -147,6 +147,8 @@ int udp_usrreq(struct socket *, > int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *); > int udp_attach(struct socket *, int); > int udp_detach(struct socket *); > +void udp_lock(struct socket *); > +void udp_unlock(struct socket *); > int udp_bind(struct socket *, struct mbuf *, struct proc *); > int udp_connect(struct socket *, struct mbuf *); > int udp_disconnect(struct socket *); > Index: netinet6/ip6_divert.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.c,v > retrieving revision 1.84 > diff -u -p -r1.84 ip6_divert.c > --- netinet6/ip6_divert.c 3 Sep 2022 18:48:50 -0000 1.84 > +++ netinet6/ip6_divert.c 3 Sep 2022 19:23:43 -0000 > @@ -67,6 +67,8 @@ const struct pr_usrreqs divert6_usrreqs > .pru_usrreq = divert6_usrreq, > .pru_attach = divert6_attach, > .pru_detach = divert6_detach, > + .pru_lock = divert6_lock, > + .pru_unlock = divert6_unlock, > .pru_bind = divert6_bind, > .pru_shutdown = divert6_shutdown, > .pru_send = divert6_send, > @@ -332,6 +334,24 @@ divert6_detach(struct socket *so) > in_pcbdetach(inp); > > return (0); > +} > + > +void > +divert6_lock(struct socket *so) > +{ > + struct inpcb *inp = sotoinpcb(so); > + > + NET_ASSERT_LOCKED(); > + mtx_enter(&inp->inp_mtx); > +} > + > +void > +divert6_unlock(struct socket *so) > +{ > + struct inpcb *inp = sotoinpcb(so); > + > + NET_ASSERT_LOCKED(); > + mtx_leave(&inp->inp_mtx); > } > > int > Index: netinet6/ip6_divert.h > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.h,v > retrieving revision 1.18 > diff -u -p -r1.18 ip6_divert.h > --- netinet6/ip6_divert.h 28 Aug 2022 18:44:17 -0000 1.18 > +++ netinet6/ip6_divert.h 3 Sep 2022 19:23:43 -0000 > @@ -74,6 +74,8 @@ int divert6_usrreq(struct socket *, > int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *); > int divert6_attach(struct socket *, int); > int divert6_detach(struct socket *); > +void divert6_lock(struct socket *); > +void divert6_unlock(struct socket *); > int divert6_bind(struct socket *, struct mbuf *, struct proc *); > int divert6_shutdown(struct socket *); > int divert6_send(struct socket *, struct mbuf *, struct mbuf *, > Index: sys/protosw.h > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/sys/protosw.h,v > retrieving revision 1.53 > diff -u -p -r1.53 protosw.h > --- sys/protosw.h 3 Sep 2022 18:48:50 -0000 1.53 > +++ sys/protosw.h 3 Sep 2022 19:23:43 -0000 > @@ -68,6 +68,8 @@ struct pr_usrreqs { > > int (*pru_attach)(struct socket *, int); > int (*pru_detach)(struct socket *); > + void (*pru_lock)(struct socket *); > + void (*pru_unlock)(struct socket *); > int (*pru_bind)(struct socket *, struct mbuf *, struct proc *); > int (*pru_listen)(struct socket *); > int (*pru_connect)(struct socket *, struct mbuf *); > @@ -277,6 +279,18 @@ static inline int > pru_detach(struct socket *so) > { > return (*so->so_proto->pr_usrreqs->pru_detach)(so); > +} > + > +static inline void > +pru_lock(struct socket *so) > +{ > + (*so->so_proto->pr_usrreqs->pru_lock)(so); > +} > + > +static inline void > +pru_unlock(struct socket *so) > +{ > + (*so->so_proto->pr_usrreqs->pru_unlock)(so); > } > > static inline int > Index: sys/socketvar.h > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/sys/socketvar.h,v > retrieving revision 1.109 > diff -u -p -r1.109 socketvar.h > --- sys/socketvar.h 1 Sep 2022 05:31:49 -0000 1.109 > +++ sys/socketvar.h 3 Sep 2022 19:23:43 -0000 > @@ -348,9 +348,11 @@ int sockargs(struct mbuf **, const void > > int sosleep_nsec(struct socket *, void *, int, const char *, uint64_t); > void solock(struct socket *); > +void solock_shared(struct socket *); > int solock_persocket(struct socket *); > void solock_pair(struct socket *, struct socket *); > void sounlock(struct socket *); > +void sounlock_shared(struct socket *); > > int sendit(struct proc *, int, struct msghdr *, int, register_t *); > int recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *); >