From 9bbbb445c5e0d62e9de4bc0c6da0319017c270a4 Mon Sep 17 00:00:00 2001 From: bluhm Date: Mon, 5 Sep 2022 14:56:08 +0000 Subject: [PATCH] Use shared netlock in soreceive(). The UDP and IP divert layer provide locking of the PCB. If that is possible, use shared instead of exclusive netlock in soreceive(). The PCB mutex provides a per socket lock against multiple soreceive() running in parallel. Release and regrab both locks in sosleep_nsec(). OK mvs@ --- sys/kern/uipc_socket.c | 20 ++++++++--------- sys/kern/uipc_socket2.c | 46 ++++++++++++++++++++++++++++++++++++++- sys/netinet/ip_divert.c | 22 ++++++++++++++++++- sys/netinet/ip_divert.h | 4 +++- sys/netinet/udp_usrreq.c | 24 +++++++++++++++++++- sys/netinet/udp_var.h | 4 +++- sys/netinet6/ip6_divert.c | 22 ++++++++++++++++++- sys/netinet6/ip6_divert.h | 4 +++- sys/sys/protosw.h | 16 +++++++++++++- sys/sys/socketvar.h | 4 +++- 10 files changed, 147 insertions(+), 19 deletions(-) diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 52a3fda1358..05e6eb58249 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uipc_socket.c,v 1.288 2022/09/04 09:04:27 bluhm Exp $ */ +/* $OpenBSD: uipc_socket.c,v 1.289 2022/09/05 14:56:08 bluhm Exp $ */ /* $NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $ */ /* @@ -822,10 +822,10 @@ bad: if (mp) *mp = NULL; - solock(so); + solock_shared(so); restart: if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) { - sounlock(so); + sounlock_shared(so); return (error); } @@ -893,7 +893,7 @@ restart: sbunlock(so, &so->so_rcv); error = sbwait(so, &so->so_rcv); if (error) { - sounlock(so); + sounlock_shared(so); return (error); } goto restart; @@ -962,11 +962,11 @@ dontblock: sbsync(&so->so_rcv, nextrecord); if (controlp) { if (pr->pr_domain->dom_externalize) { - sounlock(so); + sounlock_shared(so); error = (*pr->pr_domain->dom_externalize) (cm, controllen, flags); - solock(so); + solock_shared(so); } *controlp = cm; } else { @@ -1040,9 +1040,9 @@ dontblock: SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); resid = uio->uio_resid; - sounlock(so); + sounlock_shared(so); uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio); - solock(so); + solock_shared(so); if (uio_error) uio->uio_resid = resid - len; } else @@ -1126,7 +1126,7 @@ dontblock: error = sbwait(so, &so->so_rcv); if (error) { sbunlock(so, &so->so_rcv); - sounlock(so); + sounlock_shared(so); return (0); } if ((m = so->so_rcv.sb_mb) != NULL) @@ -1171,7 +1171,7 @@ dontblock: *flagsp |= flags; release: sbunlock(so, &so->so_rcv); - sounlock(so); + sounlock_shared(so); return (error); } diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c index 21e3a965038..3658537a02d 100644 --- a/sys/kern/uipc_socket2.c +++ b/sys/kern/uipc_socket2.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uipc_socket2.c,v 1.127 2022/08/13 21:01:46 mvs Exp $ */ +/* $OpenBSD: uipc_socket2.c,v 1.128 2022/09/05 14:56:09 bluhm Exp $ */ /* $NetBSD: uipc_socket2.c,v 1.11 1996/02/04 02:17:55 christos Exp $ */ /* @@ -360,6 +360,24 @@ solock(struct socket *so) } } +void +solock_shared(struct socket *so) +{ + switch (so->so_proto->pr_domain->dom_family) { + case PF_INET: + case PF_INET6: + if (so->so_proto->pr_usrreqs->pru_lock != NULL) { + NET_LOCK_SHARED(); + pru_lock(so); + } else + NET_LOCK(); + break; + default: + rw_enter_write(&so->so_lock); + break; + } +} + int solock_persocket(struct socket *so) { @@ -402,6 +420,24 @@ sounlock(struct socket *so) } } +void +sounlock_shared(struct socket *so) +{ + switch (so->so_proto->pr_domain->dom_family) { + case PF_INET: + case PF_INET6: + if (so->so_proto->pr_usrreqs->pru_unlock != NULL) { + pru_unlock(so); + NET_UNLOCK_SHARED(); + } else + NET_UNLOCK(); + break; + default: + rw_exit_write(&so->so_lock); + break; + } +} + void soassertlocked(struct socket *so) { @@ -425,7 +461,15 @@ sosleep_nsec(struct socket *so, void *ident, int prio, const char *wmesg, switch (so->so_proto->pr_domain->dom_family) { case PF_INET: case PF_INET6: + if (so->so_proto->pr_usrreqs->pru_unlock != NULL && + rw_status(&netlock) == RW_READ) { + pru_unlock(so); + } ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs); + if (so->so_proto->pr_usrreqs->pru_lock != NULL && + rw_status(&netlock) == RW_READ) { + pru_lock(so); + } break; default: ret = rwsleep_nsec(ident, &so->so_lock, prio, wmesg, nsecs); diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c index 361d00c4ea4..1cdad66c298 100644 --- a/sys/netinet/ip_divert.c +++ b/sys/netinet/ip_divert.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ip_divert.c,v 1.86 2022/09/03 22:43:38 mvs Exp $ */ +/* $OpenBSD: ip_divert.c,v 1.87 2022/09/05 14:56:09 bluhm Exp $ */ /* * Copyright (c) 2009 Michele Marchetto @@ -65,6 +65,8 @@ const struct sysctl_bounded_args divertctl_vars[] = { const struct pr_usrreqs divert_usrreqs = { .pru_attach = divert_attach, .pru_detach = divert_detach, + .pru_lock = divert_lock, + .pru_unlock = divert_unlock, .pru_bind = divert_bind, .pru_shutdown = divert_shutdown, .pru_send = divert_send, @@ -288,6 +290,24 @@ divert_detach(struct socket *so) return (0); } +void +divert_lock(struct socket *so) +{ + struct inpcb *inp = sotoinpcb(so); + + NET_ASSERT_LOCKED(); + mtx_enter(&inp->inp_mtx); +} + +void +divert_unlock(struct socket *so) +{ + struct inpcb *inp = sotoinpcb(so); + + NET_ASSERT_LOCKED(); + mtx_leave(&inp->inp_mtx); +} + int divert_bind(struct socket *so, struct mbuf *addr, struct proc *p) { diff --git a/sys/netinet/ip_divert.h b/sys/netinet/ip_divert.h index d5e4dabf631..5055a8f0a89 100644 --- a/sys/netinet/ip_divert.h +++ b/sys/netinet/ip_divert.h @@ -1,4 +1,4 @@ -/* $OpenBSD: ip_divert.h,v 1.21 2022/09/03 22:43:38 mvs Exp $ */ +/* $OpenBSD: ip_divert.h,v 1.22 2022/09/05 14:56:09 bluhm Exp $ */ /* * Copyright (c) 2009 Michele Marchetto @@ -72,6 +72,8 @@ void divert_packet(struct mbuf *, int, u_int16_t); int divert_sysctl(int *, u_int, void *, size_t *, void *, size_t); int divert_attach(struct socket *, int); int divert_detach(struct socket *); +void divert_lock(struct socket *); +void divert_unlock(struct socket *); int divert_bind(struct socket *, struct mbuf *, struct proc *); int divert_shutdown(struct socket *); int divert_send(struct socket *, struct mbuf *, struct mbuf *, diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c index 21440ab3a2c..a019b2c236d 100644 --- a/sys/netinet/udp_usrreq.c +++ b/sys/netinet/udp_usrreq.c @@ -1,4 +1,4 @@ -/* $OpenBSD: udp_usrreq.c,v 1.301 2022/09/03 22:43:38 mvs Exp $ */ +/* $OpenBSD: udp_usrreq.c,v 1.302 2022/09/05 14:56:09 bluhm Exp $ */ /* $NetBSD: udp_usrreq.c,v 1.28 1996/03/16 23:54:03 christos Exp $ */ /* @@ -125,6 +125,8 @@ u_int udp_recvspace = 40 * (1024 + sizeof(struct sockaddr_in)); const struct pr_usrreqs udp_usrreqs = { .pru_attach = udp_attach, .pru_detach = udp_detach, + .pru_lock = udp_lock, + .pru_unlock = udp_unlock, .pru_bind = udp_bind, .pru_connect = udp_connect, .pru_disconnect = udp_disconnect, @@ -140,6 +142,8 @@ const struct pr_usrreqs udp_usrreqs = { const struct pr_usrreqs udp6_usrreqs = { .pru_attach = udp_attach, .pru_detach = udp_detach, + .pru_lock = udp_lock, + .pru_unlock = udp_unlock, .pru_bind = udp_bind, .pru_connect = udp_connect, .pru_disconnect = udp_disconnect, @@ -1112,6 +1116,24 @@ udp_detach(struct socket *so) return (0); } +void +udp_lock(struct socket *so) +{ + struct inpcb *inp = sotoinpcb(so); + + NET_ASSERT_LOCKED(); + mtx_enter(&inp->inp_mtx); +} + +void +udp_unlock(struct socket *so) +{ + struct inpcb *inp = sotoinpcb(so); + + NET_ASSERT_LOCKED(); + mtx_leave(&inp->inp_mtx); +} + int udp_bind(struct socket *so, struct mbuf *addr, struct proc *p) { diff --git a/sys/netinet/udp_var.h b/sys/netinet/udp_var.h index 79fdfeed540..c086640cf6d 100644 --- a/sys/netinet/udp_var.h +++ b/sys/netinet/udp_var.h @@ -1,4 +1,4 @@ -/* $OpenBSD: udp_var.h,v 1.46 2022/09/03 22:43:38 mvs Exp $ */ +/* $OpenBSD: udp_var.h,v 1.47 2022/09/05 14:56:09 bluhm Exp $ */ /* $NetBSD: udp_var.h,v 1.12 1996/02/13 23:44:41 christos Exp $ */ /* @@ -145,6 +145,8 @@ int udp6_output(struct inpcb *, struct mbuf *, struct mbuf *, int udp_sysctl(int *, u_int, void *, size_t *, void *, size_t); int udp_attach(struct socket *, int); int udp_detach(struct socket *); +void udp_lock(struct socket *); +void udp_unlock(struct socket *); int udp_bind(struct socket *, struct mbuf *, struct proc *); int udp_connect(struct socket *, struct mbuf *); int udp_disconnect(struct socket *); diff --git a/sys/netinet6/ip6_divert.c b/sys/netinet6/ip6_divert.c index 706d5b9f3b9..29b3afb4642 100644 --- a/sys/netinet6/ip6_divert.c +++ b/sys/netinet6/ip6_divert.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ip6_divert.c,v 1.85 2022/09/03 22:43:38 mvs Exp $ */ +/* $OpenBSD: ip6_divert.c,v 1.86 2022/09/05 14:56:09 bluhm Exp $ */ /* * Copyright (c) 2009 Michele Marchetto @@ -66,6 +66,8 @@ const struct sysctl_bounded_args divert6ctl_vars[] = { const struct pr_usrreqs divert6_usrreqs = { .pru_attach = divert6_attach, .pru_detach = divert6_detach, + .pru_lock = divert6_lock, + .pru_unlock = divert6_unlock, .pru_bind = divert6_bind, .pru_shutdown = divert6_shutdown, .pru_send = divert6_send, @@ -295,6 +297,24 @@ divert6_detach(struct socket *so) return (0); } +void +divert6_lock(struct socket *so) +{ + struct inpcb *inp = sotoinpcb(so); + + NET_ASSERT_LOCKED(); + mtx_enter(&inp->inp_mtx); +} + +void +divert6_unlock(struct socket *so) +{ + struct inpcb *inp = sotoinpcb(so); + + NET_ASSERT_LOCKED(); + mtx_leave(&inp->inp_mtx); +} + int divert6_bind(struct socket *so, struct mbuf *addr, struct proc *p) { diff --git a/sys/netinet6/ip6_divert.h b/sys/netinet6/ip6_divert.h index d76c123e15c..df0fbcf7677 100644 --- a/sys/netinet6/ip6_divert.h +++ b/sys/netinet6/ip6_divert.h @@ -1,4 +1,4 @@ -/* $OpenBSD: ip6_divert.h,v 1.19 2022/09/03 22:43:38 mvs Exp $ */ +/* $OpenBSD: ip6_divert.h,v 1.20 2022/09/05 14:56:09 bluhm Exp $ */ /* * Copyright (c) 2009 Michele Marchetto @@ -72,6 +72,8 @@ void divert6_packet(struct mbuf *, int, u_int16_t); int divert6_sysctl(int *, u_int, void *, size_t *, void *, size_t); int divert6_attach(struct socket *, int); int divert6_detach(struct socket *); +void divert6_lock(struct socket *); +void divert6_unlock(struct socket *); int divert6_bind(struct socket *, struct mbuf *, struct proc *); int divert6_shutdown(struct socket *); int divert6_send(struct socket *, struct mbuf *, struct mbuf *, diff --git a/sys/sys/protosw.h b/sys/sys/protosw.h index eb7f67e7cb1..34eef0b441b 100644 --- a/sys/sys/protosw.h +++ b/sys/sys/protosw.h @@ -1,4 +1,4 @@ -/* $OpenBSD: protosw.h,v 1.54 2022/09/03 22:43:39 mvs Exp $ */ +/* $OpenBSD: protosw.h,v 1.55 2022/09/05 14:56:09 bluhm Exp $ */ /* $NetBSD: protosw.h,v 1.10 1996/04/09 20:55:32 cgd Exp $ */ /*- @@ -64,6 +64,8 @@ struct ifnet; struct pr_usrreqs { int (*pru_attach)(struct socket *, int); int (*pru_detach)(struct socket *); + void (*pru_lock)(struct socket *); + void (*pru_unlock)(struct socket *); int (*pru_bind)(struct socket *, struct mbuf *, struct proc *); int (*pru_listen)(struct socket *); int (*pru_connect)(struct socket *, struct mbuf *); @@ -276,6 +278,18 @@ pru_detach(struct socket *so) return (*so->so_proto->pr_usrreqs->pru_detach)(so); } +static inline void +pru_lock(struct socket *so) +{ + (*so->so_proto->pr_usrreqs->pru_lock)(so); +} + +static inline void +pru_unlock(struct socket *so) +{ + (*so->so_proto->pr_usrreqs->pru_unlock)(so); +} + static inline int pru_bind(struct socket *so, struct mbuf *nam, struct proc *p) { diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h index d21e9798325..16f586b19e0 100644 --- a/sys/sys/socketvar.h +++ b/sys/sys/socketvar.h @@ -1,4 +1,4 @@ -/* $OpenBSD: socketvar.h,v 1.109 2022/09/01 05:31:49 jsg Exp $ */ +/* $OpenBSD: socketvar.h,v 1.110 2022/09/05 14:56:09 bluhm Exp $ */ /* $NetBSD: socketvar.h,v 1.18 1996/02/09 18:25:38 christos Exp $ */ /*- @@ -348,9 +348,11 @@ int sockargs(struct mbuf **, const void *, size_t, int); int sosleep_nsec(struct socket *, void *, int, const char *, uint64_t); void solock(struct socket *); +void solock_shared(struct socket *); int solock_persocket(struct socket *); void solock_pair(struct socket *, struct socket *); void sounlock(struct socket *); +void sounlock_shared(struct socket *); int sendit(struct proc *, int, struct msghdr *, int, register_t *); int recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *); -- 2.20.1