Run raw IP input in parallel.

Running raw IPv4 input with shared net lock in parallel is less
complex than UDP.  Especially there is no socket splicing.

New ip_deliver() may run with shared or exclusive net lock.  The
last parameter indicates the mode.  If is is running with shared
netlock and encounters a protocol that needs exclusive lock, the
packet is queued.  Old ip_ours() always queued the packet.  Now it
calls ip_deliver() with shared net lock, and if that cannot handle
the packet completely, the packet is queued and later processed
with exclusive net lock.

In case of an IPv6 header chain, that switches from shared to
exclusive processing, the next protocol and mbuf offset are stored
in a mbuf tag.

OK mvs@

diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index 51c65e8..154f7de 100644 (file)
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: if_bridge.c,v 1.369 2024/02/13 12:22:09 bluhm Exp $   */
+/*     $OpenBSD: if_bridge.c,v 1.370 2024/04/14 20:46:27 bluhm Exp $   */
 
 /*
  * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net)
@@ -1600,7 +1600,7 @@ bridge_ipsec(struct ifnet *ifp, struct ether_header *eh, int hassnap,
                            off);
                        tdb_unref(tdb);
                        if (prot != IPPROTO_DONE)
-                               ip_deliver(&m, &hlen, prot, af);
+                               ip_deliver(&m, &hlen, prot, af, 0);
                        return (1);
                } else {
                        tdb_unref(tdb);

diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c
index 9748baf..a7ba54e 100644 (file)
--- a/sys/netinet/in_proto.c
+++ b/sys/netinet/in_proto.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: in_proto.c,v 1.103 2024/01/11 14:15:12 bluhm Exp $    */
+/*     $OpenBSD: in_proto.c,v 1.104 2024/04/14 20:46:27 bluhm Exp $    */
 /*     $NetBSD: in_proto.c,v 1.14 1996/02/18 18:58:32 christos Exp $   */
 
 /*
@@ -210,7 +210,7 @@ const struct protosw inetsw[] = {
   .pr_type     = SOCK_RAW,
   .pr_domain   = &inetdomain,
   .pr_protocol = IPPROTO_RAW,
-  .pr_flags    = PR_ATOMIC|PR_ADDR,
+  .pr_flags    = PR_ATOMIC|PR_ADDR|PR_MPINPUT,
   .pr_input    = rip_input,
   .pr_ctloutput        = rip_ctloutput,
   .pr_usrreqs  = &rip_usrreqs,
@@ -377,7 +377,7 @@ const struct protosw inetsw[] = {
   /* raw wildcard */
   .pr_type     = SOCK_RAW,
   .pr_domain   = &inetdomain,
-  .pr_flags    = PR_ATOMIC|PR_ADDR,
+  .pr_flags    = PR_ATOMIC|PR_ADDR|PR_MPINPUT,
   .pr_input    = rip_input,
   .pr_ctloutput        = rip_ctloutput,
   .pr_usrreqs  = &rip_usrreqs,

diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index 9c27323..25f9cc5 100644 (file)
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip_input.c,v 1.391 2024/02/28 10:57:20 bluhm Exp $    */
+/*     $OpenBSD: ip_input.c,v 1.392 2024/04/14 20:46:27 bluhm Exp $    */
 /*     $NetBSD: ip_input.c,v 1.30 1996/03/16 23:53:58 christos Exp $   */
 
 /*
@@ -245,6 +245,30 @@ ip_ours(struct mbuf **mp, int *offp, int nxt, int af)
        if (af != AF_UNSPEC)
                return nxt;
 
+       nxt = ip_deliver(mp, offp, nxt, AF_INET, 1);
+       if (nxt == IPPROTO_DONE)
+               return IPPROTO_DONE;
+
+       /* save values for later, use after dequeue */
+       if (*offp != sizeof(struct ip)) {
+               struct m_tag *mtag;
+               struct ipoffnxt *ion;
+
+               /* mbuf tags are expensive, but only used for header options */
+               mtag = m_tag_get(PACKET_TAG_IP_OFFNXT, sizeof(*ion),
+                   M_NOWAIT);
+               if (mtag == NULL) {
+                       ipstat_inc(ips_idropped);
+                       m_freemp(mp);
+                       return IPPROTO_DONE;
+               }
+               ion = (struct ipoffnxt *)(mtag + 1);
+               ion->ion_off = *offp;
+               ion->ion_nxt = nxt;
+
+               m_tag_prepend(*mp, mtag);
+       }
+
        niq_enqueue(&ipintrq, *mp);
        *mp = NULL;
        return IPPROTO_DONE;
@@ -260,18 +284,31 @@ ipintr(void)
        struct mbuf *m;
 
        while ((m = niq_dequeue(&ipintrq)) != NULL) {
-               struct ip *ip;
+               struct m_tag *mtag;
                int off, nxt;
 
 #ifdef DIAGNOSTIC
                if ((m->m_flags & M_PKTHDR) == 0)
                        panic("ipintr no HDR");
 #endif
-               ip = mtod(m, struct ip *);
-               off = ip->ip_hl << 2;
-               nxt = ip->ip_p;
+               mtag = m_tag_find(m, PACKET_TAG_IP_OFFNXT, NULL);
+               if (mtag != NULL) {
+                       struct ipoffnxt *ion;
+
+                       ion = (struct ipoffnxt *)(mtag + 1);
+                       off = ion->ion_off;
+                       nxt = ion->ion_nxt;
+
+                       m_tag_delete(m, mtag);
+               } else {
+                       struct ip *ip;
 
-               nxt = ip_deliver(&m, &off, nxt, AF_INET);
+                       ip = mtod(m, struct ip *);
+                       off = ip->ip_hl << 2;
+                       nxt = ip->ip_p;
+               }
+
+               nxt = ip_deliver(&m, &off, nxt, AF_INET, 0);
                KASSERT(nxt == IPPROTO_DONE);
        }
 }
@@ -675,15 +712,11 @@ ip_fragcheck(struct mbuf **mp, int *offp)
 #endif
 
 int
-ip_deliver(struct mbuf **mp, int *offp, int nxt, int af)
+ip_deliver(struct mbuf **mp, int *offp, int nxt, int af, int shared)
 {
-       const struct protosw *psw;
-       int naf = af;
 #ifdef INET6
        int nest = 0;
-#endif /* INET6 */
-
-       NET_ASSERT_LOCKED_EXCLUSIVE();
+#endif
 
        /*
         * Tell launch routine the next header
@@ -691,13 +724,41 @@ ip_deliver(struct mbuf **mp, int *offp, int nxt, int af)
        IPSTAT_INC(delivered);
 
        while (nxt != IPPROTO_DONE) {
+               const struct protosw *psw;
+               int naf;
+
+               switch (af) {
+               case AF_INET:
+                       psw = &inetsw[ip_protox[nxt]];
+                       break;
+#ifdef INET6
+               case AF_INET6:
+                       psw = &inet6sw[ip6_protox[nxt]];
+                       break;
+#endif
+               }
+               if (shared && !ISSET(psw->pr_flags, PR_MPINPUT)) {
+                       /* delivery not finished, decrement counter, queue */
+                       switch (af) {
+                       case AF_INET:
+                               counters_dec(ipcounters, ips_delivered);
+                               break;
+#ifdef INET6
+                       case AF_INET6:
+                               counters_dec(ip6counters, ip6s_delivered);
+                               break;
+#endif
+                       }
+                       break;
+               }
+
 #ifdef INET6
                if (af == AF_INET6 &&
                    ip6_hdrnestlimit && (++nest > ip6_hdrnestlimit)) {
                        ip6stat_inc(ip6s_toomanyhdr);
                        goto bad;
                }
-#endif /* INET6 */
+#endif
 
                /*
                 * protection against faulty packet - there should be
@@ -716,7 +777,7 @@ ip_deliver(struct mbuf **mp, int *offp, int nxt, int af)
                        }
                }
                /* Otherwise, just fall through and deliver the packet */
-#endif /* IPSEC */
+#endif
 
                switch (nxt) {
                case IPPROTO_IPV4:
@@ -728,17 +789,10 @@ ip_deliver(struct mbuf **mp, int *offp, int nxt, int af)
                        naf = AF_INET6;
                        ip6stat_inc(ip6s_delivered);
                        break;
-#endif /* INET6 */
-               }
-               switch (af) {
-               case AF_INET:
-                       psw = &inetsw[ip_protox[nxt]];
-                       break;
-#ifdef INET6
-               case AF_INET6:
-                       psw = &inet6sw[ip6_protox[nxt]];
+#endif
+               default:
+                       naf = af;
                        break;
-#endif /* INET6 */
                }
                nxt = (*psw->pr_input)(mp, offp, nxt, af);
                af = naf;

diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h
index 2c0d453..c5b9284 100644 (file)
--- a/sys/netinet/ip_var.h
+++ b/sys/netinet/ip_var.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip_var.h,v 1.114 2024/03/05 09:45:13 bluhm Exp $      */
+/*     $OpenBSD: ip_var.h,v 1.115 2024/04/14 20:46:27 bluhm Exp $      */
 /*     $NetBSD: ip_var.h,v 1.16 1996/02/13 23:43:20 christos Exp $     */
 
 /*
@@ -198,6 +198,11 @@ struct ipq {
        struct    in_addr ipq_src, ipq_dst;
 };
 
+struct ipoffnxt {
+       int     ion_off;
+       int     ion_nxt;
+};
+
 /* flags passed to ip_output */
 #define        IP_FORWARDING           0x1             /* most of ip header exists */
 #define        IP_RAWOUTPUT            0x2             /* raw ip header exists */
@@ -254,7 +259,7 @@ int  ip_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 void    ip_savecontrol(struct inpcb *, struct mbuf **, struct ip *,
            struct mbuf *);
 int     ip_input_if(struct mbuf **, int *, int, int, struct ifnet *);
-int     ip_deliver(struct mbuf **, int *, int, int);
+int     ip_deliver(struct mbuf **, int *, int, int, int);
 void    ip_forward(struct mbuf *, struct ifnet *, struct rtentry *, int);
 int     rip_ctloutput(int, struct socket *, int, int, struct mbuf *);
 void    rip_init(void);

diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index 112fe60..5f4e991 100644 (file)
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip6_input.c,v 1.259 2024/02/28 10:57:20 bluhm Exp $   */
+/*     $OpenBSD: ip6_input.c,v 1.260 2024/04/14 20:46:27 bluhm Exp $   */
 /*     $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $     */
 
 /*
@@ -166,11 +166,6 @@ ip6_init(void)
 #endif
 }
 
-struct ip6_offnxt {
-       int     ion_off;
-       int     ion_nxt;
-};
-
 /*
  * Enqueue packet for local delivery.  Queuing is used as a boundary
  * between the network layer (input/forward path) running with
@@ -190,10 +185,14 @@ ip6_ours(struct mbuf **mp, int *offp, int nxt, int af)
        if (af != AF_UNSPEC)
                return nxt;
 
+       nxt = ip_deliver(mp, offp, nxt, AF_INET6, 1);
+       if (nxt == IPPROTO_DONE)
+               return IPPROTO_DONE;
+
        /* save values for later, use after dequeue */
        if (*offp != sizeof(struct ip6_hdr)) {
                struct m_tag *mtag;
-               struct ip6_offnxt *ion;
+               struct ipoffnxt *ion;
 
                /* mbuf tags are expensive, but only used for header options */
                mtag = m_tag_get(PACKET_TAG_IP6_OFFNXT, sizeof(*ion),
@@ -203,7 +202,7 @@ ip6_ours(struct mbuf **mp, int *offp, int nxt, int af)
                        m_freemp(mp);
                        return IPPROTO_DONE;
                }
-               ion = (struct ip6_offnxt *)(mtag + 1);
+               ion = (struct ipoffnxt *)(mtag + 1);
                ion->ion_off = *offp;
                ion->ion_nxt = nxt;
 
@@ -234,9 +233,9 @@ ip6intr(void)
 #endif
                mtag = m_tag_find(m, PACKET_TAG_IP6_OFFNXT, NULL);
                if (mtag != NULL) {
-                       struct ip6_offnxt *ion;
+                       struct ipoffnxt *ion;
 
-                       ion = (struct ip6_offnxt *)(mtag + 1);
+                       ion = (struct ipoffnxt *)(mtag + 1);
                        off = ion->ion_off;
                        nxt = ion->ion_nxt;
 
@@ -248,7 +247,7 @@ ip6intr(void)
                        off = sizeof(struct ip6_hdr);
                        nxt = ip6->ip6_nxt;
                }
-               nxt = ip_deliver(&m, &off, nxt, AF_INET6);
+               nxt = ip_deliver(&m, &off, nxt, AF_INET6, 0);
                KASSERT(nxt == IPPROTO_DONE);
        }
 }

diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 866a65b..b458e0e 100644 (file)
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: mbuf.h,v 1.262 2024/02/21 13:42:06 bluhm Exp $        */
+/*     $OpenBSD: mbuf.h,v 1.263 2024/04/14 20:46:27 bluhm Exp $        */
 /*     $NetBSD: mbuf.h,v 1.19 1996/02/09 18:25:14 christos Exp $       */
 
 /*
@@ -471,6 +471,8 @@ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *);
 #define PACKET_TAG_IPSEC_IN_DONE       0x0001  /* IPsec applied, in */
 #define PACKET_TAG_IPSEC_OUT_DONE      0x0002  /* IPsec applied, out */
 #define PACKET_TAG_IPSEC_FLOWINFO      0x0004  /* IPsec flowinfo */
+#define PACKET_TAG_IP_OFFNXT           0x0010  /* IPv4 offset and next proto */
+#define PACKET_TAG_IP6_OFFNXT          0x0020  /* IPv6 offset and next proto */
 #define PACKET_TAG_WIREGUARD           0x0040  /* WireGuard data */
 #define PACKET_TAG_GRE                 0x0080  /* GRE processing done */
 #define PACKET_TAG_DLT                 0x0100 /* data link layer type */
@@ -479,7 +481,6 @@ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *);
 #define PACKET_TAG_SRCROUTE            0x1000 /* IPv4 source routing options */
 #define PACKET_TAG_TUNNEL              0x2000  /* Tunnel endpoint address */
 #define PACKET_TAG_CARP_BAL_IP         0x4000  /* carp(4) ip balanced marker */
-#define PACKET_TAG_IP6_OFFNXT          0x8000  /* IPv6 offset and next proto */
 
 #define MTAG_BITS \
     ("\20\1IPSEC_IN_DONE\2IPSEC_OUT_DONE\3IPSEC_FLOWINFO" \

diff --git a/sys/sys/protosw.h b/sys/sys/protosw.h
index c6af337..319cc5c 100644 (file)
--- a/sys/sys/protosw.h
+++ b/sys/sys/protosw.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: protosw.h,v 1.65 2024/02/03 22:50:09 mvs Exp $        */
+/*     $OpenBSD: protosw.h,v 1.66 2024/04/14 20:46:27 bluhm Exp $      */
 /*     $NetBSD: protosw.h,v 1.10 1996/04/09 20:55:32 cgd Exp $ */
 
 /*-
@@ -132,6 +132,7 @@ struct protosw {
 #define PR_ABRTACPTDIS 0x0020          /* abort on accept(2) to disconnected
                                           socket */
 #define PR_SPLICE      0x0040          /* socket splicing is possible */
+#define PR_MPINPUT     0x0080          /* input runs with shared netlock */
 
 /*
  * The arguments to usrreq are: