Use shared netlock in soreceive(). The UDP and IP divert layer

provide locking of the PCB.  If that is possible, use shared instead
of exclusive netlock in soreceive().  The PCB mutex provides a per
socket lock against multiple soreceive() running in parallel.
Release and regrab both locks in sosleep_nsec().
OK mvs@

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 52a3fda..05e6eb5 100644 (file)
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uipc_socket.c,v 1.288 2022/09/04 09:04:27 bluhm Exp $ */
+/*     $OpenBSD: uipc_socket.c,v 1.289 2022/09/05 14:56:08 bluhm Exp $ */
 /*     $NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $        */
 
 /*
@@ -822,10 +822,10 @@ bad:
        if (mp)
                *mp = NULL;
 
-       solock(so);
+       solock_shared(so);
 restart:
        if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) {
-               sounlock(so);
+               sounlock_shared(so);
                return (error);
        }
 
@@ -893,7 +893,7 @@ restart:
                sbunlock(so, &so->so_rcv);
                error = sbwait(so, &so->so_rcv);
                if (error) {
-                       sounlock(so);
+                       sounlock_shared(so);
                        return (error);
                }
                goto restart;
@@ -962,11 +962,11 @@ dontblock:
                        sbsync(&so->so_rcv, nextrecord);
                        if (controlp) {
                                if (pr->pr_domain->dom_externalize) {
-                                       sounlock(so);
+                                       sounlock_shared(so);
                                        error =
                                            (*pr->pr_domain->dom_externalize)
                                            (cm, controllen, flags);
-                                       solock(so);
+                                       solock_shared(so);
                                }
                                *controlp = cm;
                        } else {
@@ -1040,9 +1040,9 @@ dontblock:
                        SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
                        SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
                        resid = uio->uio_resid;
-                       sounlock(so);
+                       sounlock_shared(so);
                        uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
-                       solock(so);
+                       solock_shared(so);
                        if (uio_error)
                                uio->uio_resid = resid - len;
                } else
@@ -1126,7 +1126,7 @@ dontblock:
                        error = sbwait(so, &so->so_rcv);
                        if (error) {
                                sbunlock(so, &so->so_rcv);
-                               sounlock(so);
+                               sounlock_shared(so);
                                return (0);
                        }
                        if ((m = so->so_rcv.sb_mb) != NULL)
@@ -1171,7 +1171,7 @@ dontblock:
                *flagsp |= flags;
 release:
        sbunlock(so, &so->so_rcv);
-       sounlock(so);
+       sounlock_shared(so);
        return (error);
 }
 

diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index 21e3a96..3658537 100644 (file)
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uipc_socket2.c,v 1.127 2022/08/13 21:01:46 mvs Exp $  */
+/*     $OpenBSD: uipc_socket2.c,v 1.128 2022/09/05 14:56:09 bluhm Exp $        */
 /*     $NetBSD: uipc_socket2.c,v 1.11 1996/02/04 02:17:55 christos Exp $       */
 
 /*
@@ -360,6 +360,24 @@ solock(struct socket *so)
        }
 }
 
+void
+solock_shared(struct socket *so)
+{
+       switch (so->so_proto->pr_domain->dom_family) {
+       case PF_INET:
+       case PF_INET6:
+               if (so->so_proto->pr_usrreqs->pru_lock != NULL) {
+                       NET_LOCK_SHARED();
+                       pru_lock(so);
+               } else
+                       NET_LOCK();
+               break;
+       default:
+               rw_enter_write(&so->so_lock);
+               break;
+       }
+}
+
 int
 solock_persocket(struct socket *so)
 {
@@ -402,6 +420,24 @@ sounlock(struct socket *so)
        }
 }
 
+void
+sounlock_shared(struct socket *so)
+{
+       switch (so->so_proto->pr_domain->dom_family) {
+       case PF_INET:
+       case PF_INET6:
+               if (so->so_proto->pr_usrreqs->pru_unlock != NULL) {
+                       pru_unlock(so);
+                       NET_UNLOCK_SHARED();
+               } else
+                       NET_UNLOCK();
+               break;
+       default:
+               rw_exit_write(&so->so_lock);
+               break;
+       }
+}
+
 void
 soassertlocked(struct socket *so)
 {
@@ -425,7 +461,15 @@ sosleep_nsec(struct socket *so, void *ident, int prio, const char *wmesg,
        switch (so->so_proto->pr_domain->dom_family) {
        case PF_INET:
        case PF_INET6:
+               if (so->so_proto->pr_usrreqs->pru_unlock != NULL &&
+                   rw_status(&netlock) == RW_READ) {
+                       pru_unlock(so);
+               }
                ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs);
+               if (so->so_proto->pr_usrreqs->pru_lock != NULL &&
+                   rw_status(&netlock) == RW_READ) {
+                       pru_lock(so);
+               }
                break;
        default:
                ret = rwsleep_nsec(ident, &so->so_lock, prio, wmesg, nsecs);

diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index 361d00c..1cdad66 100644 (file)
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.c,v 1.86 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip_divert.c,v 1.87 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -65,6 +65,8 @@ const struct sysctl_bounded_args divertctl_vars[] = {
 const struct pr_usrreqs divert_usrreqs = {
        .pru_attach     = divert_attach,
        .pru_detach     = divert_detach,
+       .pru_lock       = divert_lock,
+       .pru_unlock     = divert_unlock,
        .pru_bind       = divert_bind,
        .pru_shutdown   = divert_shutdown,
        .pru_send       = divert_send,
@@ -288,6 +290,24 @@ divert_detach(struct socket *so)
        return (0);
 }
 
+void
+divert_lock(struct socket *so)
+{
+       struct inpcb *inp = sotoinpcb(so);
+
+       NET_ASSERT_LOCKED();
+       mtx_enter(&inp->inp_mtx);
+}
+
+void
+divert_unlock(struct socket *so)
+{
+       struct inpcb *inp = sotoinpcb(so);
+
+       NET_ASSERT_LOCKED();
+       mtx_leave(&inp->inp_mtx);
+}
+
 int
 divert_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {

diff --git a/sys/netinet/ip_divert.h b/sys/netinet/ip_divert.h
index d5e4dab..5055a8f 100644 (file)
--- a/sys/netinet/ip_divert.h
+++ b/sys/netinet/ip_divert.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.h,v 1.21 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip_divert.h,v 1.22 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -72,6 +72,8 @@ void   divert_packet(struct mbuf *, int, u_int16_t);
 int     divert_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int     divert_attach(struct socket *, int);
 int     divert_detach(struct socket *);
+void    divert_lock(struct socket *);
+void    divert_unlock(struct socket *);
 int     divert_bind(struct socket *, struct mbuf *, struct proc *);
 int     divert_shutdown(struct socket *);
 int     divert_send(struct socket *, struct mbuf *, struct mbuf *,

diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 21440ab..a019b2c 100644 (file)
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: udp_usrreq.c,v 1.301 2022/09/03 22:43:38 mvs Exp $    */
+/*     $OpenBSD: udp_usrreq.c,v 1.302 2022/09/05 14:56:09 bluhm Exp $  */
 /*     $NetBSD: udp_usrreq.c,v 1.28 1996/03/16 23:54:03 christos Exp $ */
 
 /*
@@ -125,6 +125,8 @@ u_int       udp_recvspace = 40 * (1024 + sizeof(struct sockaddr_in));
 const struct pr_usrreqs udp_usrreqs = {
        .pru_attach     = udp_attach,
        .pru_detach     = udp_detach,
+       .pru_lock       = udp_lock,
+       .pru_unlock     = udp_unlock,
        .pru_bind       = udp_bind,
        .pru_connect    = udp_connect,
        .pru_disconnect = udp_disconnect,
@@ -140,6 +142,8 @@ const struct pr_usrreqs udp_usrreqs = {
 const struct pr_usrreqs udp6_usrreqs = {
        .pru_attach     = udp_attach,
        .pru_detach     = udp_detach,
+       .pru_lock       = udp_lock,
+       .pru_unlock     = udp_unlock,
        .pru_bind       = udp_bind,
        .pru_connect    = udp_connect,
        .pru_disconnect = udp_disconnect,
@@ -1112,6 +1116,24 @@ udp_detach(struct socket *so)
        return (0);
 }
 
+void
+udp_lock(struct socket *so)
+{
+       struct inpcb *inp = sotoinpcb(so);
+
+       NET_ASSERT_LOCKED();
+       mtx_enter(&inp->inp_mtx);
+}
+
+void
+udp_unlock(struct socket *so)
+{
+       struct inpcb *inp = sotoinpcb(so);
+
+       NET_ASSERT_LOCKED();
+       mtx_leave(&inp->inp_mtx);
+}
+
 int
 udp_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {

diff --git a/sys/netinet/udp_var.h b/sys/netinet/udp_var.h
index 79fdfee..c086640 100644 (file)
--- a/sys/netinet/udp_var.h
+++ b/sys/netinet/udp_var.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: udp_var.h,v 1.46 2022/09/03 22:43:38 mvs Exp $        */
+/*     $OpenBSD: udp_var.h,v 1.47 2022/09/05 14:56:09 bluhm Exp $      */
 /*     $NetBSD: udp_var.h,v 1.12 1996/02/13 23:44:41 christos Exp $    */
 
 /*
@@ -145,6 +145,8 @@ int  udp6_output(struct inpcb *, struct mbuf *, struct mbuf *,
 int     udp_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int     udp_attach(struct socket *, int);
 int     udp_detach(struct socket *);
+void    udp_lock(struct socket *);
+void    udp_unlock(struct socket *);
 int     udp_bind(struct socket *, struct mbuf *, struct proc *);
 int     udp_connect(struct socket *, struct mbuf *);
 int     udp_disconnect(struct socket *);

diff --git a/sys/netinet6/ip6_divert.c b/sys/netinet6/ip6_divert.c
index 706d5b9..29b3afb 100644 (file)
--- a/sys/netinet6/ip6_divert.c
+++ b/sys/netinet6/ip6_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip6_divert.c,v 1.85 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip6_divert.c,v 1.86 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -66,6 +66,8 @@ const struct sysctl_bounded_args divert6ctl_vars[] = {
 const struct pr_usrreqs divert6_usrreqs = {
        .pru_attach     = divert6_attach,
        .pru_detach     = divert6_detach,
+       .pru_lock       = divert6_lock,
+       .pru_unlock     = divert6_unlock,
        .pru_bind       = divert6_bind,
        .pru_shutdown   = divert6_shutdown,
        .pru_send       = divert6_send,
@@ -295,6 +297,24 @@ divert6_detach(struct socket *so)
        return (0);
 }
 
+void
+divert6_lock(struct socket *so)
+{
+       struct inpcb *inp = sotoinpcb(so);
+
+       NET_ASSERT_LOCKED();
+       mtx_enter(&inp->inp_mtx);
+}
+
+void
+divert6_unlock(struct socket *so)
+{
+       struct inpcb *inp = sotoinpcb(so);
+
+       NET_ASSERT_LOCKED();
+       mtx_leave(&inp->inp_mtx);
+}
+
 int
 divert6_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {

diff --git a/sys/netinet6/ip6_divert.h b/sys/netinet6/ip6_divert.h
index d76c123..df0fbcf 100644 (file)
--- a/sys/netinet6/ip6_divert.h
+++ b/sys/netinet6/ip6_divert.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip6_divert.h,v 1.19 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip6_divert.h,v 1.20 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -72,6 +72,8 @@ void   divert6_packet(struct mbuf *, int, u_int16_t);
 int     divert6_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int     divert6_attach(struct socket *, int);
 int     divert6_detach(struct socket *);
+void    divert6_lock(struct socket *);
+void    divert6_unlock(struct socket *);
 int     divert6_bind(struct socket *, struct mbuf *, struct proc *);
 int     divert6_shutdown(struct socket *);
 int     divert6_send(struct socket *, struct mbuf *, struct mbuf *,

diff --git a/sys/sys/protosw.h b/sys/sys/protosw.h
index eb7f67e..34eef0b 100644 (file)
--- a/sys/sys/protosw.h
+++ b/sys/sys/protosw.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: protosw.h,v 1.54 2022/09/03 22:43:39 mvs Exp $        */
+/*     $OpenBSD: protosw.h,v 1.55 2022/09/05 14:56:09 bluhm Exp $      */
 /*     $NetBSD: protosw.h,v 1.10 1996/04/09 20:55:32 cgd Exp $ */
 
 /*-
@@ -64,6 +64,8 @@ struct ifnet;
 struct pr_usrreqs {
        int     (*pru_attach)(struct socket *, int);
        int     (*pru_detach)(struct socket *);
+       void    (*pru_lock)(struct socket *);
+       void    (*pru_unlock)(struct socket *);
        int     (*pru_bind)(struct socket *, struct mbuf *, struct proc *);
        int     (*pru_listen)(struct socket *);
        int     (*pru_connect)(struct socket *, struct mbuf *);
@@ -276,6 +278,18 @@ pru_detach(struct socket *so)
        return (*so->so_proto->pr_usrreqs->pru_detach)(so);
 }
 
+static inline void
+pru_lock(struct socket *so)
+{
+       (*so->so_proto->pr_usrreqs->pru_lock)(so);
+}
+
+static inline void
+pru_unlock(struct socket *so)
+{
+       (*so->so_proto->pr_usrreqs->pru_unlock)(so);
+}
+
 static inline int
 pru_bind(struct socket *so, struct mbuf *nam, struct proc *p)
 {

diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index d21e979..16f586b 100644 (file)
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: socketvar.h,v 1.109 2022/09/01 05:31:49 jsg Exp $     */
+/*     $OpenBSD: socketvar.h,v 1.110 2022/09/05 14:56:09 bluhm Exp $   */
 /*     $NetBSD: socketvar.h,v 1.18 1996/02/09 18:25:38 christos Exp $  */
 
 /*-
@@ -348,9 +348,11 @@ int        sockargs(struct mbuf **, const void *, size_t, int);
 
 int    sosleep_nsec(struct socket *, void *, int, const char *, uint64_t);
 void   solock(struct socket *);
+void   solock_shared(struct socket *);
 int    solock_persocket(struct socket *);
 void   solock_pair(struct socket *, struct socket *);
 void   sounlock(struct socket *);
+void   sounlock_shared(struct socket *);
 
 int    sendit(struct proc *, int, struct msghdr *, int, register_t *);
 int    recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *);